text-sanctifier 1.3.0 → 1.3.2

package/src/hiddenText.js

@@ -0,0 +1,119 @@
+// src/hiddenText.js
+//
+// Finds runs of the characters used to hide instructions in text a model reads:
+// tag characters, variation selectors and invisible operators. Tag runs are decoded
+// to the ASCII they spell. Used by `text-sanctifier check --ai-config`.
+//
+// Node and command-line only: src/index.js does not import this file, so it never
+// reaches the browser bundle.
+
+/** @typedef {'tag' | 'variation_selector' | 'invisible_operator'} HiddenRunKind */
+
+/**
+ * @typedef {Object} HiddenRun
+ * @property {HiddenRunKind} kind
+ * @property {number} offset - UTF-16 index of the first character
+ * @property {number} end - UTF-16 index just past the last character
+ * @property {number} count - number of code points in the run
+ * @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText)
+ */
+
+/**
+ * @typedef {Object} HiddenTextScan
+ * @property {HiddenRun[]} runs - runs that may hide text, in order
+ * @property {Array<[number, number]>} benign - [offset, end) spans of legitimate
+ *   sequences (subdivision flags, a single emoji or text presentation selector)
+ */
+
+const TAG_FIRST = 0xE0000;
+const TAG_LAST = 0xE007F;
+const CANCEL_TAG = 0xE007F;
+const BLACK_FLAG = 0x1F3F4;
+
+// RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
+const SUBDIVISION_FLAGS = new Set(['gbeng', 'gbsct', 'gbwls']);
+
+/**
+ * @param {number} cp
+ * @returns {HiddenRunKind | null}
+ */
+function kindOf(cp) {
+    if (cp >= TAG_FIRST && cp <= TAG_LAST) return 'tag';
+    if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
+        (cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
+    if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
+    return null;
+}
+
+/** @param {number} cp */
+const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
+
+/**
+ * Decodes tag characters to the ASCII they shadow: U+E0020–U+E007E become
+ * U+0020–U+007E. Anything else (U+E0001 LANGUAGE TAG, U+E007F CANCEL TAG, unassigned
+ * tags, other characters) is written as `<U+XXXX>`, so the result is printable.
+ * @param {string} text
+ * @returns {string}
+ */
+export function decodeTagText(text) {
+    let out = '';
+    for (const ch of text) {
+        const cp = /** @type {number} */ (ch.codePointAt(0));
+        out += cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
+    }
+    return out;
+}
+
+/**
+ * Finds runs of tag characters, variation selectors and invisible operators.
+ * Consecutive characters of the same kind form one run. Two kinds of sequence are
+ * legitimate and listed under `benign` instead of `runs`:
+ *  - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
+ *    tags for "gbeng", "gbsct" or "gbwls" and U+E007F
+ *  - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
+ * @param {string} text
+ * @returns {HiddenTextScan}
+ */
+export function findHiddenRuns(text) {
+    /** @type {HiddenRun[]} */
+    const runs = [];
+    /** @type {Array<[number, number]>} */
+    const benign = [];
+
+    let i = 0;
+    while (i < text.length) {
+        const cp = /** @type {number} */ (text.codePointAt(i));
+        const kind = kindOf(cp);
+        if (kind === null) { i += cp > 0xFFFF ? 2 : 1; continue; }
+
+        const offset = i;
+        let count = 0;
+        let last = cp;
+        while (i < text.length) {
+            const c = /** @type {number} */ (text.codePointAt(i));
+            if (kindOf(c) !== kind) break;
+            last = c;
+            count++;
+            i += c > 0xFFFF ? 2 : 1;
+        }
+        const end = i;
+
+        if (kind === 'tag') {
+            const body = text.slice(offset, end);
+            const afterFlag = offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG;
+            if (afterFlag && last === CANCEL_TAG && SUBDIVISION_FLAGS.has(decodeTagText(body).slice(0, -'<U+E007F>'.length))) {
+                benign.push([offset, end]);
+            } else {
+                runs.push({ kind, offset, end, count, decoded: decodeTagText(body) });
+            }
+            continue;
+        }
+        if (kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
+            offset > 0 && !/\s/.test(text[offset - 1])) {
+            benign.push([offset, end]);
+            continue;
+        }
+        runs.push({ kind, offset, end, count });
+    }
+    return { runs, benign };
+}

package/src/index.d.ts

@@ -1,12 +1,14 @@
 import { inspectText } from './inspectText.js';
 import { inspectTextDetailed } from './inspectText.js';
+import { getNewlineStats } from './inspectText.js';
 import { getRecommendedSanctifierOptions } from './inspectText.js';
 import { summonSanctifier } from './sanctifyText.js';
-export { inspectText, inspectTextDetailed, getRecommendedSanctifierOptions, summonSanctifier };
+export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier };
 export type UnicodeTrashReport = import('./inspectText.js').UnicodeTrashReport;
 export type TrashCategory = import('./inspectText.js').TrashCategory;
 export type TrashFinding = import('./inspectText.js').TrashFinding;
 export type TrashCounts = import('./inspectText.js').TrashCounts;
 export type DetailedTrashReport = import('./inspectText.js').DetailedTrashReport;
 export type InspectDetailedOptions = import('./inspectText.js').InspectDetailedOptions;
+export type NewlineStats = import('./inspectText.js').NewlineStats;
 export type SanctifyOptions = import('./sanctifyText.js').SanctifyOptions;

package/src/index.js

@@ -1,8 +1,8 @@
 // src/index.js
 
 
-import { inspectText, inspectTextDetailed, getRecommendedSanctifierOptions } from './inspectText.js';
-export { inspectText, inspectTextDetailed, getRecommendedSanctifierOptions };
+import { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions } from './inspectText.js';
+export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions };
 
 import { summonSanctifier } from './sanctifyText.js';
 export { summonSanctifier };
@@ -14,6 +14,7 @@ export { summonSanctifier };
 /** @typedef {import('./inspectText.js').TrashCounts} TrashCounts */
 /** @typedef {import('./inspectText.js').DetailedTrashReport} DetailedTrashReport */
 /** @typedef {import('./inspectText.js').InspectDetailedOptions} InspectDetailedOptions */
+/** @typedef {import('./inspectText.js').NewlineStats} NewlineStats */
 /** @typedef {import('./sanctifyText.js').SanctifyOptions} SanctifyOptions */
 
 

package/src/inspectText.js

@@ -7,6 +7,7 @@ import {
     ASCII_KEYBOARD_SAFE_REGEX,
     normalizeTypographicJank,
 } from './sanctifyText.js';
+import { classifyCodeUnit, classifyAstral } from './categories.generated.js';
 
 /** @typedef {import('./sanctifyText.js').SanctifyOptions} SanctifyOptions */
 
@@ -167,73 +168,26 @@ export function inspectText(text) {
 /**
  * @typedef {Object} InspectDetailedOptions
  * @property {number} [maxFindings=100] - Cap on listed findings (non-negative integer or Infinity).
+ *   Each listed finding is an object, so memory grows with the number listed; 0 builds none (counts only).
  */
 
 const DEFAULT_MAX_FINDINGS = 100;
 
-/**
- * Category of one BMP code point (one UTF-16 code unit), or null when it is not trash.
- * Tab, LF and CR are whitespace, not trash. DEL (U+007F) is counted with the C0 controls.
- * Surrogates return null; the caller pairs them and uses classifyAstral.
- * @param {number} c
- * @returns {TrashCategory|null}
- */
-function classifyCodeUnit(c) {
-    if (c < 0x80) {
-        if (c === 0) return 'nul';
-        if (c < 0x20) return c === 0x09 || c === 0x0A || c === 0x0D ? null : 'c0_control';
-        return c === 0x7F ? 'c0_control' : null;
-    }
-    if (c <= 0x9F) return 'c1_control';
-    if (c === 0xA0) return 'nonbreaking_space';
-    if (c === 0xAD || c === 0x034F || c === 0x180E) return 'invisible_format';
-    if (c === 0x061C) return 'bidi_control';
-    if (c === 0x115F || c === 0x1160) return 'hangul_filler';
-    if (c < 0x2000) return null;
-    if (c <= 0x200A) return 'nonbreaking_space';
-    if (c <= 0x200D) return 'zero_width';
-    if (c <= 0x200F) return 'bidi_control';
-    if (c >= 0x202A && c <= 0x202E) return 'bidi_control';
-    if (c === 0x202F) return 'nonbreaking_space';
-    if (c === 0x2060) return 'zero_width';
-    if (c >= 0x2061 && c <= 0x2064) return 'invisible_format';
-    if (c >= 0x2066 && c <= 0x2069) return 'bidi_control';
-    if (c === 0x3000) return 'nonbreaking_space';
-    if (c === 0x3164 || c === 0xFFA0) return 'hangul_filler';
-    if (c >= 0xFE00 && c <= 0xFE0F) return 'variation_selector';
-    if (c === 0xFEFF) return 'bom';
-    return null;
-}
-
-/**
- * Category of a code point above U+FFFF, or null when it is not trash.
- * @param {number} cp
- * @returns {TrashCategory|null}
- */
-function classifyAstral(cp) {
-    if (cp >= 0xE0000 && cp <= 0xE007F) return 'tag';
-    if (cp >= 0xE0100 && cp <= 0xE01EF) return 'variation_selector';
-    return null;
-}
-
-/** High surrogate of U+E0000–U+E03FF, the only astral block with trash (tags, VS17–VS256). */
-const TRASH_HIGH_SURROGATE = 0xDB40;
-
 /**
  * Locates invisible and control characters, one finding per character.
  *
- * Categories:
+ * Categories (generated from scripts/categories.js by `npm run gen`):
  * - `bidi_control`: U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069
  * - `zero_width`: U+200B–U+200D, U+2060
- * - `nonbreaking_space`: non-breaking and typographic spaces U+00A0, U+2000–U+200A, U+202F, U+3000
- * - `bom`: U+FEFF (`atStart` says whether it sits at offset 0)
+ * - `nonbreaking_space`: U+00A0, U+2000–U+200A, U+202F, U+205F, U+3000 (non-breaking and typographic spaces)
+ * - `bom`: U+FEFF (`atStart` is true at offset 0, where a BOM is legitimate)
  * - `nul`: U+0000
- * - `c0_control`: U+0001–U+001F except tab, LF and CR; and U+007F
+ * - `c0_control`: U+0001–U+0008, U+000B, U+000C, U+000E–U+001F, U+007F (tab, LF and CR are whitespace, not trash)
  * - `c1_control`: U+0080–U+009F
- * - `invisible_format`: soft hyphen U+00AD, U+034F, U+180E, invisible operators U+2061–U+2064
+ * - `invisible_format`: U+00AD (soft hyphen), U+034F (combining grapheme joiner), U+17B4, U+17B5 (Khmer inherent vowels), U+180E (Mongolian vowel separator), U+2061–U+2064 (invisible operators), U+2065 (reserved), U+206A–U+206F (deprecated format characters), U+FFF0–U+FFF8 (reserved), U+1BCA0–U+1BCA3 (shorthand format controls), U+1D173–U+1D17A (musical formatting), U+E0080–U+E00FF (reserved), U+E01F0–U+E0FFF (reserved) (every other Default_Ignorable_Code_Point, which renderers draw as nothing)
  * - `hangul_filler`: U+115F, U+1160, U+3164, U+FFA0 (letters that render blank)
- * - `variation_selector`: U+FE00–U+FE0F, U+E0100–U+E01EF (including VS16 in emoji)
- * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling")
+ * - `variation_selector`: U+180B–U+180D (Mongolian FVS1-FVS3), U+180F (Mongolian FVS4), U+FE00–U+FE0F (VS1-VS16, including VS16 in emoji), U+E0100–U+E01EF (VS17-VS256)
+ * - `tag`: U+E0000–U+E007F (tag characters, used for "ASCII smuggling" of hidden instructions)
  *
  * `codePoint` is the full code point; `offset` and `column` count UTF-16 units.
  * Every occurrence is counted; only the first `maxFindings` are listed.
@@ -297,7 +251,7 @@ export function inspectTextDetailed(text, options = {}) {
         const offset = i;
         let codePoint = c;
         let category;
-        if (c === TRASH_HIGH_SURROGATE) {
+        if (c >= 0xD800 && c <= 0xDBFF) {
             const low = text.charCodeAt(i + 1);
             if (!(low >= 0xDC00 && low <= 0xDFFF)) continue; // lone surrogate
             codePoint = 0x10000 + ((c - 0xD800) << 10) + (low - 0xDC00);
@@ -322,15 +276,18 @@ export function inspectTextDetailed(text, options = {}) {
 }
 
 /**
- * Counts the number of different newline types in a string.
+ * @typedef {Object} NewlineStats
+ * @property {number} crlf - CR LF pairs.
+ * @property {number} cr - CRs not followed by LF.
+ * @property {number} lf - LFs not preceded by CR.
+ * @property {('CRLF'|'CR'|'LF')[]} types - Styles present, in the order CRLF, CR, LF.
+ * @property {boolean} mixed - True when more than one style is present.
+ */
+
+/**
+ * Counts each newline style in one pass, with no regex and no copy.
  * @param {string} text
- * @returns {{
- *   crlf: number,
- *   cr: number,
- *   lf: number,
- *   types: string[],
- *   mixed: boolean
- * }}
+ * @returns {NewlineStats}
  */
 export function getNewlineStats(text) {
     if (typeof text !== 'string') {

package/src/sanctifyText.js

@@ -1,5 +1,8 @@
 // src/sanctifyText.js
 
+import { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX } from './categories.generated.js';
+export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX };
+
 
 /**
  * @typedef {Object} SanctifyOptions
@@ -272,21 +275,14 @@ export function sanctifyText(
 
 
 /**
- * Purges invisible Unicode "trash" characters and removes them.
- * 
- * Targets:
- * - Non-breaking spaces ( )
- * - Zero-width spaces and miscellaneous Unicode spaces ( –‍,  , ⁠, 　, )
- * - Bidi controls: marks, embeddings/overrides and isolates (؜, ‎, ‏, ‪–‮, ⁦–⁩)
- * - Invisible format characters: soft hyphen, CGJ, Mongolian vowel separator, invisible operators (­, ͏, ᠎, ⁡–⁤)
- * - Hangul fillers (ᅟ, ᅠ, ㅤ, ﾠ)
- * - Variation selectors (︀–️, \u{E0100}–\u{E01EF}), including VS16 in emoji
- * - Tag characters (\u{E0000}–\u{E007F}), used for "ASCII smuggling"
+ * Removes the invisible Unicode characters in INVISIBLE_TRASH_REGEX: non-breaking and
+ * typographic spaces, zero-width characters, bidi controls, invisible format characters,
+ * Hangul fillers, variation selectors (including VS16 in emoji), the BOM and tag characters.
+ * The code points come from the table in scripts/categories.js.
  *
  * @param {string} text
  * @returns {string}
  */
-export const INVISIBLE_TRASH_REGEX = /[ ­͏؜ᅟᅠ᠎ -‏‪- ⁠-⁤⁦-⁩　ㅤ︀-️ﾠ\u{E0000}-\u{E007F}\u{E0100}-\u{E01EF}]+/gu;
 function purgeInvisibleTrash(text) {
     return text.replace(INVISIBLE_TRASH_REGEX, '');
 }
@@ -497,15 +493,14 @@ function collapseExtraSpaces(text) {
  * Nukes hidden control characters that are invisible and often dangerous.
  * (Excludes necessary whitespace like \n and \t.)
  * 
- * Control characters nuked: 
- *   - ASCII control range (0x00-0x1F, 0x7F)
- *   - Unicode control range (0x80-0x9F)
- *   - Bidi controls (U+061C, U+200E, U+200F, U+202A–U+202E, U+2066–U+2069)
+ * Control characters nuked (the code points come from the table in scripts/categories.js):
+ *   - NUL and the C0 controls except tab, LF and CR; DEL (U+007F)
+ *   - C1 controls (U+0080-U+009F)
+ *   - Bidi controls
  * 
  * @param {string} text
  * @returns {string}
  */
-export const CONTROL_CHARS_REGEX = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F\u0080-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g;
 function purgeControlCharacters(text) {
     return text.replace(CONTROL_CHARS_REGEX, '');
 }