text-sanctifier 1.3.1 → 1.3.3

package/dist/text-sanctifier.min.js

@@ -1,14 +1,15 @@
-function n(a={}){const c=!!a.purgeInvisibleChars,d=!!a.purgeEmojis,e=!!a.nukeControls,g=!!a.keyboardOnlyFilter,f=!!a.normalizeNewlines,l=!!a.trimSpacesAroundNewlines,b=!!a.collapseNewLines,h=!!a.preserveParagraphs,k=!!a.collapseSpaces,r=!!a.finalTrim,m=Number.isFinite(a.maxLength)&&0<=a.maxLength?a.maxLength:Infinity,p="truncate"===a.g||"noop"===a.g||"throw"===a.g?a.g:"throw";return A=>q(A,c,d,e,g,f,l,b,h,k,r,m,p)}n.strict=a=>q(a,!0,!0,!0,!1,!0,!0,!0,!1,!0,!0);
-n.loose=a=>q(a,!1,!1,!1,!1,!0,!0,!0,!0,!0,!0);n.keyboardOnlyEmoji=a=>q(a,!1,!1,!1,!0,!0,!0,!1,!1,!1,!0);n.keyboardOnly=a=>q(a,!0,!0,!0,!0,!0,!0,!0,!1,!0,!0);
-function q(a,c=!1,d=!1,e=!1,g=!1,f=!1,l=!1,b=!1,h=!1,k=!1,r=!1,m=Infinity,p="throw"){if("string"!==typeof a)throw new TypeError("sanctifyText expects a string input.");"truncate"!==p&&"noop"!==p&&"throw"!==p&&(p="throw");if(a.length>m)switch(p){case "truncate":a=a.slice(0,m);break;case "noop":return a;default:throw new RangeError(`sanctifyText input length ${a.length} exceeds maxLength ${m}.`);}c&&(a=a.replace(t,""));d&&(a=a.replace(u,""));e&&(a=a.replace(v,""));g&&(a=w(a,d));f&&(a=a.replace(x,"\n"));
-l&&(a=a.replace(y,"$1"));b&&(c=a,a=h?c.replace(z,"\n\n"):c.replace(B,"\n"));k&&(a=a.replace(C," "));return r?a.trim():a}var t=/[\u00a0\u00ad\u034f\u061c\u115f\u1160\u180e\u2000-\u200f\u202a-\u202f\u205f\u2060-\u2064\u2066-\u2069\u3000\u3164\ufe00-\ufe0f\ufeff\uffa0\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]+/gu,D=/[^\x20-\x7E\r\n]+/gu;
-function w(a,c=!1){a=E(a);return c?a.replace(D,""):a.replace(D,d=>{u.lastIndex=0;if(u.test(d)){u.lastIndex=0;var e="";for(const g of d.matchAll(u))e+=g[0];d=e}else d="";return d})}var F=/[\u2018\u2019\u201A\u201B\u2032\u2035]/g,G=/[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g,H=/[\u2012\u2013\u2014\u2015\u2212]/g,I=/\u2026/g,J=/[\u2022\u00B7]/g,K=/[\uFF01-\uFF5E]/g;
-function E(a){return a.replace(F,"'").replace(G,'"').replace(H,"-").replace(I,"...").replace(J,"*").replace(K,c=>String.fromCharCode(c.charCodeAt(0)-65248))}var u;try{u=RegExp("(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)","gu")}catch{u=/[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu}var x=/\r\n|\r|\n/g,y=/[ \t]*(\n+)[ \t]*/g,B=/\n{2,}/g,z=/\n{3,}/g,C=/ {2,}/g,v=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F\u0080-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g;
-function L(a,c){a.lastIndex=0;return a.test(c)}
-function M(a){if("string"!==typeof a)throw new TypeError("inspectText expects a string input.");const c=[],d={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:c};L(v,a)&&(d.hasControlChars=!0,c.push("Control characters detected."));L(t,a)&&(d.hasInvisibleChars=!0,c.push("Invisible Unicode characters detected."));L(u,a)&&(d.hasEmojis=!0,c.push("Emojis detected."));const {mixed:e,types:g}=N(a);d.hasMixedNewlines=e;d.newlineStyle=
-e?"Mixed":g[0]||null;d.newlineStyle&&c.push(e?"Mixed newline styles detected.":`Consistent newline style: ${d.newlineStyle}`);a:{a=E(a);if(L(D,a))for(f of a)if(!("\n"===f||"\r"===f||" "<=f&&"~">=f||L(u,f))){var f=!0;break a}f=!1}f&&(d.hasNonKeyboardChars=!0,c.push("Non-keyboard characters detected."));return d}
-function O(a,c={}){if("string"!==typeof a)throw new TypeError("inspectTextDetailed expects a string input.");c=c&&c.maxFindings;c=Infinity===c||Number.isInteger(c)&&0<=c?c:100;const d={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},e=[];let g=0,f=1,l=0;for(let k=0,r=a.length;k<r;k++){var b=a.charCodeAt(k);if(32<=b&&127>b)continue;if(10===b){f++;l=k+1;continue}if(13===b){10!==a.charCodeAt(k+1)&&(f++,
-l=k+1);continue}const m=k;var h=b;if(56128===b){h=a.charCodeAt(k+1);if(!(56320<=h&&57343>=h))continue;h=65536+(b-55296<<10)+(h-56320);b=917504<=h&&917631>=h?"tag":917760<=h&&917999>=h?"variation_selector":null;k++}else b=128>b?0===b?"nul":32>b?9===b||10===b||13===b?null:"c0_control":127===b?"c0_control":null:159>=b?"c1_control":160===b?"nonbreaking_space":173===b||847===b||6158===b?"invisible_format":1564===b?"bidi_control":4447===b||4448===b?"hangul_filler":8192>b?null:8202>=b?"nonbreaking_space":
-8205>=b?"zero_width":8207>=b||8234<=b&&8238>=b?"bidi_control":8239===b||8287===b?"nonbreaking_space":8288===b?"zero_width":8289<=b&&8292>=b?"invisible_format":8294<=b&&8297>=b?"bidi_control":12288===b?"nonbreaking_space":12644===b||65440===b?"hangul_filler":65024<=b&&65039>=b?"variation_selector":65279===b?"bom":null;null!==b&&(d[b]++,g++,e.length<c&&(h={offset:m,line:f,column:m-l+1,codePoint:h,category:b},"bom"===b&&(h.atStart=0===m),e.push(h)))}return{findings:e,counts:d,total:g,truncated:g>e.length}}
-function N(a){if("string"!==typeof a)throw new TypeError("getNewlineStats expects a string input.");var c=0;let d=0,e=0;for(let g=0,f=a.length;g<f;g++){const l=a.charCodeAt(g);13===l?10===a.charCodeAt(g+1)?(c++,g++):d++:10===l&&e++}a={crlf:c,cr:d,lf:e};c=[];0<a.crlf&&c.push("CRLF");0<a.cr&&c.push("CR");0<a.lf&&c.push("LF");return{...a,types:c,mixed:1<c.length}}
-function P(a){return{purgeInvisibleChars:a.hasInvisibleChars,purgeEmojis:a.hasEmojis,nukeControls:a.hasControlChars,keyboardOnlyFilter:a.hasNonKeyboardChars,normalizeNewlines:a.hasMixedNewlines||"CRLF"===a.newlineStyle||"CR"===a.newlineStyle}}export { n as summonSanctifier, M as inspectText, P as getRecommendedSanctifierOptions, O as inspectTextDetailed, N as getNewlineStats };
+var n=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g,q=/[\u00A0\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180B-\u180F\u2000-\u200F\u202A-\u202F\u205F-\u206F\u3000\u3164\uFE00-\uFE0F\uFEFF\uFFA0\uFFF0-\uFFF8\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0000}-\u{E0FFF}]+/gu;
+function t(a={}){const c=!!a.purgeInvisibleChars,d=!!a.purgeEmojis,f=!!a.nukeControls,h=!!a.keyboardOnlyFilter,g=!!a.normalizeNewlines,l=!!a.trimSpacesAroundNewlines,b=!!a.collapseNewLines,e=!!a.preserveParagraphs,k=!!a.collapseSpaces,r=!!a.finalTrim,m=Number.isFinite(a.maxLength)&&0<=a.maxLength?a.maxLength:Infinity,p="truncate"===a.g||"noop"===a.g||"throw"===a.g?a.g:"throw";return A=>u(A,c,d,f,h,g,l,b,e,k,r,m,p)}t.strict=a=>u(a,!0,!0,!0,!1,!0,!0,!0,!1,!0,!0);
+t.loose=a=>u(a,!1,!1,!1,!1,!0,!0,!0,!0,!0,!0);t.keyboardOnlyEmoji=a=>u(a,!1,!1,!1,!0,!0,!0,!1,!1,!1,!0);t.keyboardOnly=a=>u(a,!0,!0,!0,!0,!0,!0,!0,!1,!0,!0);
+function u(a,c=!1,d=!1,f=!1,h=!1,g=!1,l=!1,b=!1,e=!1,k=!1,r=!1,m=Infinity,p="throw"){if("string"!==typeof a)throw new TypeError("sanctifyText expects a string input.");"truncate"!==p&&"noop"!==p&&"throw"!==p&&(p="throw");if(a.length>m)switch(p){case "truncate":a=a.slice(0,m);break;case "noop":return a;default:throw new RangeError(`sanctifyText input length ${a.length} exceeds maxLength ${m}.`);}c&&(a=a.replace(q,""));d&&(a=a.replace(v,""));f&&(a=a.replace(n,""));h&&(a=w(a,d));g&&(a=a.replace(x,"\n"));
+l&&(a=a.replace(y,"$1"));b&&(c=a,a=e?c.replace(z,"\n\n"):c.replace(B,"\n"));k&&(a=a.replace(C," "));return r?a.trim():a}var D=/[^\x20-\x7E\r\n]+/gu;function w(a,c=!1){a=E(a);return c?a.replace(D,""):a.replace(D,d=>{v.lastIndex=0;if(v.test(d)){v.lastIndex=0;var f="";for(const h of d.matchAll(v))f+=h[0];d=f}else d="";return d})}
+var F=/[\u2018\u2019\u201A\u201B\u2032\u2035]/g,G=/[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g,H=/[\u2012\u2013\u2014\u2015\u2212]/g,I=/\u2026/g,J=/[\u2022\u00B7]/g,K=/[\uFF01-\uFF5E]/g;function E(a){return a.replace(F,"'").replace(G,'"').replace(H,"-").replace(I,"...").replace(J,"*").replace(K,c=>String.fromCharCode(c.charCodeAt(0)-65248))}var v;try{v=RegExp("(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)","gu")}catch{v=/[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu}
+var x=/\r\n|\r|\n/g,y=/[ \t]*(\n+)[ \t]*/g,B=/\n{2,}/g,z=/\n{3,}/g,C=/ {2,}/g;function L(a,c){a.lastIndex=0;return a.test(c)}
+function M(a){if("string"!==typeof a)throw new TypeError("inspectText expects a string input.");const c=[],d={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:c};L(n,a)&&(d.hasControlChars=!0,c.push("Control characters detected."));L(q,a)&&(d.hasInvisibleChars=!0,c.push("Invisible Unicode characters detected."));L(v,a)&&(d.hasEmojis=!0,c.push("Emojis detected."));const {mixed:f,types:h}=N(a);d.hasMixedNewlines=f;d.newlineStyle=
+f?"Mixed":h[0]||null;d.newlineStyle&&c.push(f?"Mixed newline styles detected.":`Consistent newline style: ${d.newlineStyle}`);a:{a=E(a);if(L(D,a))for(g of a)if(!("\n"===g||"\r"===g||" "<=g&&"~">=g||L(v,g))){var g=!0;break a}g=!1}g&&(d.hasNonKeyboardChars=!0,c.push("Non-keyboard characters detected."));return d}
+function O(a,c={}){if("string"!==typeof a)throw new TypeError("inspectTextDetailed expects a string input.");c=c&&c.maxFindings;c=Infinity===c||Number.isInteger(c)&&0<=c?c:100;const d={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},f=[];let h=0,g=1,l=0;for(let k=0,r=a.length;k<r;k++){var b=a.charCodeAt(k);if(32<=b&&127>b)continue;if(10===b){g++;l=k+1;continue}if(13===b){10!==a.charCodeAt(k+1)&&(g++,
+l=k+1);continue}const m=k;var e=b;if(55296<=b&&56319>=b){e=a.charCodeAt(k+1);if(!(56320<=e&&57343>=e))continue;e=65536+(b-55296<<10)+(e-56320);b=917504>e?113828>e?113824>e?null:"invisible_format":119155>e?null:119163>e?"invisible_format":null:917760>e?917632>e?"tag":"invisible_format":918E3>e?"variation_selector":921600>e?"invisible_format":null;k++}else b=8192>b?174>b?32>b?11>b?1>b?"nul":9>b?"c0_control":null:13>b?"c0_control":14>b?null:"c0_control":160>b?127>b?null:128>b?"c0_control":"c1_control":
+161>b?"nonbreaking_space":173>b?null:"invisible_format":4449>b?1564>b?847>b?null:848>b?"invisible_format":null:1565>b?"bidi_control":4447>b?null:"hangul_filler":6155>b?6068>b?null:6070>b?"invisible_format":null:6159>b?6158>b?"variation_selector":"invisible_format":6160>b?"variation_selector":null:8304>b?8240>b?8208>b?8203>b?"nonbreaking_space":8206>b?"zero_width":"bidi_control":8234>b?null:8239>b?"bidi_control":"nonbreaking_space":8289>b?8287>b?null:8288>b?"nonbreaking_space":"zero_width":8294>b?
+"invisible_format":8298>b?"bidi_control":"invisible_format":65040>b?12644>b?12288>b?null:12289>b?"nonbreaking_space":null:12645>b?"hangul_filler":65024>b?null:"variation_selector":65440>b?65279>b?null:65280>b?"bom":null:65520>b?65441>b?"hangul_filler":null:65529>b?"invisible_format":null;null!==b&&(d[b]++,h++,f.length<c&&(e={offset:m,line:g,column:m-l+1,codePoint:e,category:b},"bom"===b&&(e.atStart=0===m),f.push(e)))}return{findings:f,counts:d,total:h,truncated:h>f.length}}
+function N(a){if("string"!==typeof a)throw new TypeError("getNewlineStats expects a string input.");var c=0;let d=0,f=0;for(let h=0,g=a.length;h<g;h++){const l=a.charCodeAt(h);13===l?10===a.charCodeAt(h+1)?(c++,h++):d++:10===l&&f++}a={crlf:c,cr:d,lf:f};c=[];0<a.crlf&&c.push("CRLF");0<a.cr&&c.push("CR");0<a.lf&&c.push("LF");return{...a,types:c,mixed:1<c.length}}
+function P(a){return{purgeInvisibleChars:a.hasInvisibleChars,purgeEmojis:a.hasEmojis,nukeControls:a.hasControlChars,keyboardOnlyFilter:a.hasNonKeyboardChars,normalizeNewlines:a.hasMixedNewlines||"CRLF"===a.newlineStyle||"CR"===a.newlineStyle}}export { t as summonSanctifier, M as inspectText, P as getRecommendedSanctifierOptions, O as inspectTextDetailed, N as getNewlineStats };

package/dist/types/categories.generated.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Category of one BMP code point (one UTF-16 code unit), or null when it is not trash.
+ * Surrogates return null; the caller pairs them and uses classifyAstral.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyCodeUnit(c: number): TrashCategory | null;
+/**
+ * Category of a code point above U+FFFF, or null when it is not trash.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyAstral(c: number): TrashCategory | null;
+/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
+/**
+ * Removed by `nukeControls`. Categories: bidi_control, nul, c0_control, c1_control.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const CONTROL_CHARS_REGEX: RegExp;
+/**
+ * Removed by `purgeInvisibleChars`. Categories: bidi_control, zero_width, nonbreaking_space, bom, invisible_format, hangul_filler, variation_selector, tag.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const INVISIBLE_TRASH_REGEX: RegExp;
+export type TrashCategory = import("./inspectText.js").TrashCategory;

package/dist/types/hiddenText.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Decodes tag characters to the ASCII they shadow: U+E0020–U+E007E become
+ * U+0020–U+007E. Anything else (U+E0001 LANGUAGE TAG, U+E007F CANCEL TAG, unassigned
+ * tags, other characters) is written as `<U+XXXX>`, so the result is printable.
+ * @param {string} text
+ * @returns {string}
+ */
+export function decodeTagText(text: string): string;
+/**
+ * Finds runs of tag characters, variation selectors and invisible operators.
+ * Consecutive characters of the same kind form one run. Two kinds of sequence are
+ * legitimate and listed under `benign` instead of `runs`:
+ *  - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
+ *    tags for "gbeng", "gbsct" or "gbwls" and U+E007F
+ *  - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
+ * @param {string} text
+ * @returns {HiddenTextScan}
+ */
+export function findHiddenRuns(text: string): HiddenTextScan;
+export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator";
+export type HiddenRun = {
+    kind: HiddenRunKind;
+    /**
+     * - UTF-16 index of the first character
+     */
+    offset: number;
+    /**
+     * - UTF-16 index just past the last character
+     */
+    end: number;
+    /**
+     * - number of code points in the run
+     */
+    count: number;
+    /**
+     * - tag runs only: the ASCII the tags spell (see decodeTagText)
+     */
+    decoded?: string;
+};
+export type HiddenTextScan = {
+    /**
+     * - runs that may hide text, in order
+     */
+    runs: HiddenRun[];
+    /**
+     * - [offset, end) spans of legitimate
+     * sequences (subdivision flags, a single emoji or text presentation selector)
+     */
+    benign: Array<[number, number]>;
+};

package/dist/types/inspectText.d.ts

@@ -6,18 +6,18 @@ export function inspectText(text: string): UnicodeTrashReport;
 /**
  * Locates invisible and control characters, one finding per character.
  *
- * Categories:
+ * Categories (generated from scripts/categories.js by `npm run gen`):
  * - `bidi_control`: U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069
  * - `zero_width`: U+200B–U+200D, U+2060
- * - `nonbreaking_space`: non-breaking and typographic spaces U+00A0, U+2000–U+200A, U+202F, U+205F, U+3000
- * - `bom`: U+FEFF (`atStart` says whether it sits at offset 0)
+ * - `nonbreaking_space`: U+00A0, U+2000–U+200A, U+202F, U+205F, U+3000 (non-breaking and typographic spaces)
+ * - `bom`: U+FEFF (`atStart` is true at offset 0, where a BOM is legitimate)
  * - `nul`: U+0000
- * - `c0_control`: U+0001–U+001F except tab, LF and CR; and U+007F
+ * - `c0_control`: U+0001–U+0008, U+000B, U+000C, U+000E–U+001F, U+007F (tab, LF and CR are whitespace, not trash)
  * - `c1_control`: U+0080–U+009F
- * - `invisible_format`: soft hyphen U+00AD, U+034F, U+180E, invisible operators U+2061–U+2064
+ * - `invisible_format`: U+00AD (soft hyphen), U+034F (combining grapheme joiner), U+17B4, U+17B5 (Khmer inherent vowels), U+180E (Mongolian vowel separator), U+2061–U+2064 (invisible operators), U+2065 (reserved), U+206A–U+206F (deprecated format characters), U+FFF0–U+FFF8 (reserved), U+1BCA0–U+1BCA3 (shorthand format controls), U+1D173–U+1D17A (musical formatting), U+E0080–U+E00FF (reserved), U+E01F0–U+E0FFF (reserved) (every other Default_Ignorable_Code_Point, which renderers draw as nothing)
  * - `hangul_filler`: U+115F, U+1160, U+3164, U+FFA0 (letters that render blank)
- * - `variation_selector`: U+FE00–U+FE0F, U+E0100–U+E01EF (including VS16 in emoji)
- * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling")
+ * - `variation_selector`: U+180B–U+180D (Mongolian FVS1-FVS3), U+180F (Mongolian FVS4), U+FE00–U+FE0F (VS1-VS16, including VS16 in emoji), U+E0100–U+E01EF (VS17-VS256)
+ * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling" of hidden instructions)
  *
  * `codePoint` is the full code point; `offset` and `column` count UTF-16 units.
  * Every occurrence is counted; only the first `maxFindings` are listed.

package/dist/types/sanctifyText.d.ts

@@ -117,22 +117,6 @@ export function sanctifyText(text: string, purgeInvisibleChars?: boolean, purgeE
  * @returns {string}
  */
 export function normalizeTypographicJank(text: string): string;
-/**
- * Purges invisible Unicode "trash" characters and removes them.
- *
- * Targets:
- * - Non-breaking spaces ( )
- * - Zero-width spaces and miscellaneous Unicode spaces ( –‍,  ,  , ⁠, 　, )
- * - Bidi controls: marks, embeddings/overrides and isolates (؜, ‎, ‏, ‪–‮, ⁦–⁩)
- * - Invisible format characters: soft hyphen, CGJ, Mongolian vowel separator, invisible operators (­, ͏, ᠎, ⁡–⁤)
- * - Hangul fillers (ᅟ, ᅠ, ㅤ, ﾠ)
- * - Variation selectors (︀–️, \u{E0100}–\u{E01EF}), including VS16 in emoji
- * - Tag characters (\u{E0000}–\u{E007F}), used for "ASCII smuggling"
- *
- * @param {string} text
- * @returns {string}
- */
-export const INVISIBLE_TRASH_REGEX: RegExp;
 /**
  * Matches any character that is NOT:
  * - Printable ASCII (U+0020–U+007E)
@@ -142,19 +126,6 @@ export const INVISIBLE_TRASH_REGEX: RegExp;
 export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
 /** @type {RegExp} */
 export let EMOJI_REGEX: RegExp;
-/**
- * Nukes hidden control characters that are invisible and often dangerous.
- * (Excludes necessary whitespace like \n and \t.)
- *
- * Control characters nuked:
- *   - ASCII control range (0x00-0x1F, 0x7F)
- *   - Unicode control range (0x80-0x9F)
- *   - Bidi controls (U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069)
- *
- * @param {string} text
- * @returns {string}
- */
-export const CONTROL_CHARS_REGEX: RegExp;
 export type SanctifyOptions = {
     purgeInvisibleChars?: boolean;
     purgeEmojis?: boolean;
@@ -169,3 +140,6 @@ export type SanctifyOptions = {
     maxLength?: number;
     onMaxLength?: "throw" | "truncate" | "noop";
 };
+import { CONTROL_CHARS_REGEX } from './categories.generated.js';
+import { INVISIBLE_TRASH_REGEX } from './categories.generated.js';
+export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX };

package/package.json

@@ -1,74 +1,80 @@
-{
-  "name": "text-sanctifier",
-  "version": "1.3.1",
-  "type": "module",
-  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
-  "engines": {
-    "node": ">=18"
-  },
-  "main": "./src/index.js",
-  "module": "./src/index.js",
-  "browser": "./dist/text-sanctifier.min.js",
-  "bin": {
-    "text-sanctifier": "bin/text-sanctifier.js"
-  },
-  "files": [
-    "src",
-    "bin",
-    "dist/text-sanctifier.min.js",
-    "dist/types",
-    "LICENSE",
-    "README.md",
-    "ADDITIONAL_TERMS.md",
-    "robots.txt"
-  ],
-  "exports": {
-    ".": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./src/index.js"
-    },
-    "./browser": {
-      "types": "./dist/types/index.d.ts",
-      "default": "./dist/text-sanctifier.min.js"
-    }
-  },
-  "types": "./dist/types/index.d.ts",
-  "sideEffects": false,
-  "scripts": {
-    "build": "node scripts/buildc.js",
-    "build:types": "tsc -p tsconfig.types.json",
-    "test:types": "tsc -p tests/types/tsconfig.json",
-    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node tests/cli.test.js",
-    "test-min": "node tests/sanctifyText.test.min.js",
-    "ci": "npm run test && npm run build && npm run build:types && npm run test:types && npm run test-min"
-  },
-  "keywords": [
-    "text",
-    "sanitize",
-    "normalize",
-    "invisible",
-    "scrubber",
-    "unicode",
-    "clean",
-    "sanctify"
-  ],
-  "author": "WATT3D <WATT3D@protonmail.com>",
-  "license": "SEE LICENSE IN LICENSE",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
-  },
-  "bugs": {
-    "url": "https://github.com/iWhatty/text-sanctifier/issues"
-  },
-  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
-  "directories": {
-    "test": "tests"
-  },
-  "devDependencies": {
-    "esbuild": "^0.25.3",
-    "google-closure-compiler": "^20240317.0.0",
-    "terser": "^5.39.0",
-    "typescript": "^5.9.3"
-  }
-}
+{
+  "name": "text-sanctifier",
+  "version": "1.3.3",
+  "type": "module",
+  "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
+  "engines": {
+    "node": ">=18"
+  },
+  "main": "./src/index.js",
+  "module": "./src/index.js",
+  "browser": "./dist/text-sanctifier.min.js",
+  "bin": {
+    "text-sanctifier": "bin/text-sanctifier.js"
+  },
+  "files": [
+    "src",
+    "bin",
+    "dist/text-sanctifier.min.js",
+    "dist/types",
+    "LICENSE",
+    "README.md",
+    "ADDITIONAL_TERMS.md",
+    "robots.txt"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./src/index.js"
+    },
+    "./hidden": {
+      "types": "./dist/types/hiddenText.d.ts",
+      "default": "./src/hiddenText.js"
+    },
+    "./browser": {
+      "types": "./dist/types/index.d.ts",
+      "default": "./dist/text-sanctifier.min.js"
+    }
+  },
+  "types": "./dist/types/index.d.ts",
+  "sideEffects": false,
+  "scripts": {
+    "gen": "node scripts/gen-categories.js",
+    "gen:check": "node scripts/gen-categories.js --check",
+    "build": "node scripts/buildc.js",
+    "build:types": "tsc -p tsconfig.types.json",
+    "test:types": "tsc -p tests/types/tsconfig.json",
+    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node tests/cli.test.js",
+    "test-min": "node tests/sanctifyText.test.min.js",
+    "ci": "npm run gen:check && npm run test && npm run build && npm run build:types && npm run test:types && npm run test-min"
+  },
+  "keywords": [
+    "text",
+    "sanitize",
+    "normalize",
+    "invisible",
+    "scrubber",
+    "unicode",
+    "clean",
+    "sanctify"
+  ],
+  "author": "WATT3D <WATT3D@protonmail.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/iWhatty/text-sanctifier.git"
+  },
+  "bugs": {
+    "url": "https://github.com/iWhatty/text-sanctifier/issues"
+  },
+  "homepage": "https://github.com/iWhatty/text-sanctifier#readme",
+  "directories": {
+    "test": "tests"
+  },
+  "devDependencies": {
+    "esbuild": "^0.25.3",
+    "google-closure-compiler": "^20240317.0.0",
+    "terser": "^5.39.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/categories.generated.js

@@ -0,0 +1,223 @@
+// src/categories.generated.js
+//
+// Generated by scripts/gen-categories.js from the table in scripts/categories.js.
+// Do not edit: change the table and run `npm run gen`. `npm run ci` fails while this file is stale.
+
+/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
+
+/**
+ * Removed by `nukeControls`. Categories: bidi_control, nul, c0_control, c1_control.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const CONTROL_CHARS_REGEX = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g;
+
+/**
+ * Removed by `purgeInvisibleChars`. Categories: bidi_control, zero_width, nonbreaking_space, bom, invisible_format, hangul_filler, variation_selector, tag.
+ * Global: reset `lastIndex` before calling `.test()`.
+ * @type {RegExp}
+ */
+export const INVISIBLE_TRASH_REGEX = /[\u00A0\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180B-\u180F\u2000-\u200F\u202A-\u202F\u205F-\u206F\u3000\u3164\uFE00-\uFE0F\uFEFF\uFFA0\uFFF0-\uFFF8\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0000}-\u{E0FFF}]+/gu;
+
+/**
+ * Category of one BMP code point (one UTF-16 code unit), or null when it is not trash.
+ * Surrogates return null; the caller pairs them and uses classifyAstral.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyCodeUnit(c) {
+    return c < 0x2000 ? classifyBelow2000(c) : classifyFrom2000(c);
+}
+
+// Two halves rather than one tree: V8 inlines a function only while its bytecode is
+// small (460 bytes by default), and the single pass is much faster when it does.
+
+/** @param {number} c @returns {TrashCategory|null} */
+function classifyBelow2000(c) {
+    if (c < 0x00AE) {
+        if (c < 0x0020) {
+            if (c < 0x000B) {
+                if (c < 0x0001) {
+                    return 'nul';
+                }
+                if (c < 0x0009) {
+                    return 'c0_control';
+                }
+                return null;
+            }
+            if (c < 0x000D) {
+                return 'c0_control';
+            }
+            if (c < 0x000E) {
+                return null;
+            }
+            return 'c0_control';
+        }
+        if (c < 0x00A0) {
+            if (c < 0x007F) {
+                return null;
+            }
+            if (c < 0x0080) {
+                return 'c0_control';
+            }
+            return 'c1_control';
+        }
+        if (c < 0x00A1) {
+            return 'nonbreaking_space';
+        }
+        if (c < 0x00AD) {
+            return null;
+        }
+        return 'invisible_format';
+    }
+    if (c < 0x1161) {
+        if (c < 0x061C) {
+            if (c < 0x034F) {
+                return null;
+            }
+            if (c < 0x0350) {
+                return 'invisible_format';
+            }
+            return null;
+        }
+        if (c < 0x061D) {
+            return 'bidi_control';
+        }
+        if (c < 0x115F) {
+            return null;
+        }
+        return 'hangul_filler';
+    }
+    if (c < 0x180B) {
+        if (c < 0x17B4) {
+            return null;
+        }
+        if (c < 0x17B6) {
+            return 'invisible_format';
+        }
+        return null;
+    }
+    if (c < 0x180F) {
+        if (c < 0x180E) {
+            return 'variation_selector';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0x1810) {
+        return 'variation_selector';
+    }
+    return null;
+}
+
+/** @param {number} c @returns {TrashCategory|null} */
+function classifyFrom2000(c) {
+    if (c < 0x2070) {
+        if (c < 0x2030) {
+            if (c < 0x2010) {
+                if (c < 0x200B) {
+                    return 'nonbreaking_space';
+                }
+                if (c < 0x200E) {
+                    return 'zero_width';
+                }
+                return 'bidi_control';
+            }
+            if (c < 0x202A) {
+                return null;
+            }
+            if (c < 0x202F) {
+                return 'bidi_control';
+            }
+            return 'nonbreaking_space';
+        }
+        if (c < 0x2061) {
+            if (c < 0x205F) {
+                return null;
+            }
+            if (c < 0x2060) {
+                return 'nonbreaking_space';
+            }
+            return 'zero_width';
+        }
+        if (c < 0x2066) {
+            return 'invisible_format';
+        }
+        if (c < 0x206A) {
+            return 'bidi_control';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0xFE10) {
+        if (c < 0x3164) {
+            if (c < 0x3000) {
+                return null;
+            }
+            if (c < 0x3001) {
+                return 'nonbreaking_space';
+            }
+            return null;
+        }
+        if (c < 0x3165) {
+            return 'hangul_filler';
+        }
+        if (c < 0xFE00) {
+            return null;
+        }
+        return 'variation_selector';
+    }
+    if (c < 0xFFA0) {
+        if (c < 0xFEFF) {
+            return null;
+        }
+        if (c < 0xFF00) {
+            return 'bom';
+        }
+        return null;
+    }
+    if (c < 0xFFF0) {
+        if (c < 0xFFA1) {
+            return 'hangul_filler';
+        }
+        return null;
+    }
+    if (c < 0xFFF9) {
+        return 'invisible_format';
+    }
+    return null;
+}
+
+/**
+ * Category of a code point above U+FFFF, or null when it is not trash.
+ * @param {number} c
+ * @returns {TrashCategory|null}
+ */
+export function classifyAstral(c) {
+    if (c < 0xE0000) {
+        if (c < 0x1BCA4) {
+            if (c < 0x1BCA0) {
+                return null;
+            }
+            return 'invisible_format';
+        }
+        if (c < 0x1D173) {
+            return null;
+        }
+        if (c < 0x1D17B) {
+            return 'invisible_format';
+        }
+        return null;
+    }
+    if (c < 0xE0100) {
+        if (c < 0xE0080) {
+            return 'tag';
+        }
+        return 'invisible_format';
+    }
+    if (c < 0xE01F0) {
+        return 'variation_selector';
+    }
+    if (c < 0xE1000) {
+        return 'invisible_format';
+    }
+    return null;
+}