text-sanctifier 1.3.6 → 1.3.7

package/README.md

@@ -264,8 +264,30 @@ const { runs, benign, counts, total, truncated } = findHiddenRuns(text, { maxRun
 - `maxRuns` caps the `runs` list and, separately, the `benign` list; `counts`, `total` and `benignTotal` count everything, and `truncated` is true when either list was cut. `maxDecodedLength` caps each `decoded` string in UTF-16 units and never cuts a `<U+XXXX>` marker. Both default to no cap, and an invalid value (negative, fractional, a string) also means no cap, so set them explicitly for untrusted input. With both set, memory stays flat on any input: on 16 M characters (32 MB) of hidden text the heap grows by under 1 MB, against hundreds of MB with no caps.
 - `decodeTagText` turns U+E0020–U+E007E into the ASCII they shadow and writes any other character as `<U+XXXX>`.
 - Zero-width, bidi and other invisible characters are not runs; use `inspectTextDetailed` for those.
+- `codePointCounts` has the same keys as `counts` and counts the code points in those runs, so long runs weigh more than short ones; `supplementarySelectorCount` counts U+E0100–U+E01EF on their own, apart from the emoji selectors U+FE00–U+FE0F. Both leave out benign spans and stay exact under `maxRuns`.
 - `VERSION` is exported here and from the package root: the installed version, as in `package.json`.
 
+### `text-sanctifier/context`: `inspectTextInContext(text, options?)`
+
+`inspectTextDetailed` with context. Some invisible characters are part of ordinary text: the joiners in a family emoji, the selector in ✌ followed by VS16, an ideographic variation selector after a CJK character. This entry point reports the same findings and `counts`, marks each listed finding `benign: true` or `false`, and adds `benignCounts`, exact whether or not a finding is listed. The categories are the same.
+
+```js
+import { inspectTextInContext } from 'text-sanctifier/context';
+
+const report = inspectTextInContext(text, { maxFindings: 20 });
+const suspicious = report.counts.zero_width - report.benignCounts.zero_width; // U+200B, U+2060, stray U+200D
+```
+
+Benign, and nothing else:
+
+- U+200D between two emoji (`Extended_Pictographic`), allowing a skin-tone modifier or VS16 after the first
+- U+FE0E or U+FE0F directly after an emoji, or in a keycap (0-9, # or *, the selector, U+20E3)
+- one of U+E0100–U+E01EF directly after an ideograph
+- U+180B–U+180D or U+180F directly after a Mongolian letter
+- the tags of the England, Scotland and Wales flags
+
+U+200B, U+2060, U+FEFF, bidi controls and all other tags are never benign. Joiners in scripts that need them (ZWNJ in Persian, ZWJ in Indic scripts) are not benign yet. It needs Unicode property escapes (Node 10+, current browsers), is not in the browser file, and is about 1.7 KB gzipped on its own.
+
 ---
 
 ## Command line
@@ -334,7 +356,7 @@ The repository publishes a [pre-commit](https://pre-commit.com) hook manifest, s
 # .pre-commit-config.yaml
 repos:
   - repo: https://github.com/iWhatty/text-sanctifier
-    rev: v1.3.6  # a release tag
+    rev: v1.3.7  # a release tag
     hooks:
       - id: text-sanctifier            # every staged text file
       - id: text-sanctifier-ai-config  # agent instruction files, with --ai-config

package/dist/text-sanctifier.min.js

@@ -3,4 +3,4 @@ var y=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u2
 `){return e.replace(Q,r)}var re=/[ \t]*(\n+)[ \t]*/g;function ne(e){return e.replace(re,"$1")}var te=/\n{2,}/g,ie=/\n{3,}/g;function oe(e,r){return r?e.replace(ie,`
 
 `):e.replace(te,`
-`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.6";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};
+`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.7";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};

package/dist/types/context.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Locates invisible and control characters as inspectTextDetailed does, and marks
+ * the ones that belong to legitimate sequences as benign:
+ * - U+200D between two emoji (Extended_Pictographic), allowing a skin-tone modifier or
+ *   VS16 after the first: an emoji ZWJ sequence
+ * - U+FE0E or U+FE0F directly after an emoji, or between 0-9, # or * and U+20E3 (keycap)
+ * - one of U+E0100–U+E01EF directly after an ideograph: an ideographic variation sequence
+ * - U+180B–U+180D or U+180F directly after a Mongolian letter
+ * - the tags of the England, Scotland and Wales flags after U+1F3F4
+ * Nothing else is benign: not U+200B, U+2060, U+FEFF, bidi controls or other tags.
+ *
+ * `counts`, `total` and `truncated` are exactly those of inspectTextDetailed;
+ * `benignCounts` counts every benign character, listed or not. Two passes over the
+ * string, no copy; memory is flat with `maxFindings` set.
+ *
+ * @param {string} text
+ * @param {InspectDetailedOptions} [options]
+ * @returns {ContextReport}
+ */
+export function inspectTextInContext(text: string, options?: InspectDetailedOptions): ContextReport;
+export type TrashFinding = import("./inspectText.js").TrashFinding;
+export type TrashCounts = import("./inspectText.js").TrashCounts;
+export type InspectDetailedOptions = import("./inspectText.js").InspectDetailedOptions;
+/**
+ * A finding with `benign`: true when the character is part of a legitimate sequence.
+ */
+export type ContextFinding = TrashFinding & {
+    benign: boolean;
+};
+export type ContextReport = {
+    /**
+     * - As in inspectTextDetailed, each with `benign`.
+     */
+    findings: ContextFinding[];
+    /**
+     * - As in inspectTextDetailed: every occurrence, benign or not.
+     */
+    counts: TrashCounts;
+    /**
+     * - Sum of `counts`.
+     */
+    total: number;
+    /**
+     * - True when `total` exceeds `findings.length`.
+     */
+    truncated: boolean;
+    /**
+     * - Benign occurrences per category, listed or not.
+     * `counts[c] - benignCounts[c]` is the number of suspicious characters in category `c`.
+     */
+    benignCounts: TrashCounts;
+    /**
+     * - Sum of `benignCounts`.
+     */
+    benignTotal: number;
+};

package/dist/types/hiddenText.d.ts

@@ -79,6 +79,15 @@ export type HiddenTextScan = {
      * - every run per kind, listed or not
      */
     counts: HiddenRunCounts;
+    /**
+     * - code points in those runs per kind, listed or not
+     */
+    codePointCounts: HiddenRunCounts;
+    /**
+     * - code points U+E0100–U+E01EF in those runs, which
+     * (unlike the emoji selectors U+FE00–U+FE0F) can carry data; included in codePointCounts.variation_selector
+     */
+    supplementarySelectorCount: number;
     /**
      * - sum of `counts`
      */

package/dist/types/version.generated.d.ts

@@ -1,2 +1,2 @@
 /** The installed version of text-sanctifier, as in its package.json. */
-export const VERSION: "1.3.6";
+export const VERSION: "1.3.7";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "text-sanctifier",
-  "version": "1.3.6",
+  "version": "1.3.7",
   "type": "module",
   "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
   "engines": {
@@ -31,6 +31,10 @@
       "types": "./dist/types/hiddenText.d.ts",
       "default": "./src/hiddenText.js"
     },
+    "./context": {
+      "types": "./dist/types/context.d.ts",
+      "default": "./src/context.js"
+    },
     "./browser": {
       "types": "./dist/types/index.d.ts",
       "default": "./dist/text-sanctifier.min.js"
@@ -44,7 +48,7 @@
     "build": "node scripts/build.js",
     "build:types": "tsc -p tsconfig.types.json",
     "test:types": "tsc -p tests/types/tsconfig.json",
-    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node tests/cli.test.js",
+    "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js",
     "test-min": "node tests/sanctifyText.test.min.js",
     "ci": "npm run gen:check && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
     "size": "node scripts/check-size.js"

package/src/context.js

@@ -0,0 +1,177 @@
+// src/context.js
+//
+// inspectTextInContext: inspectTextDetailed plus which findings belong to legitimate
+// sequences (emoji joiners and selectors, ideographic variation sequences, Mongolian
+// selectors, subdivision flags), counted exactly per category. The categories are
+// unchanged; see "Proposal: context-aware findings" in DESIGN.md.
+//
+// Exported as `text-sanctifier/context`. It needs Unicode property escapes in regexes,
+// so it lives apart from the core and the browser file and has its own size budget.
+
+import { inspectTextDetailed } from './inspectText.js';
+
+/** @typedef {import('./inspectText.js').TrashFinding} TrashFinding */
+/** @typedef {import('./inspectText.js').TrashCounts} TrashCounts */
+/** @typedef {import('./inspectText.js').InspectDetailedOptions} InspectDetailedOptions */
+
+/**
+ * @typedef {TrashFinding & { benign: boolean }} ContextFinding
+ * A finding with `benign`: true when the character is part of a legitimate sequence.
+ */
+
+/**
+ * @typedef {Object} ContextReport
+ * @property {ContextFinding[]} findings - As in inspectTextDetailed, each with `benign`.
+ * @property {TrashCounts} counts - As in inspectTextDetailed: every occurrence, benign or not.
+ * @property {number} total - Sum of `counts`.
+ * @property {boolean} truncated - True when `total` exceeds `findings.length`.
+ * @property {TrashCounts} benignCounts - Benign occurrences per category, listed or not.
+ *   `counts[c] - benignCounts[c]` is the number of suspicious characters in category `c`.
+ * @property {number} benignTotal - Sum of `benignCounts`.
+ */
+
+// Tested at an index of the text itself, so nothing is allocated per character.
+const PICTOGRAPHIC_AT = /\p{Extended_Pictographic}/yu;
+const IDEOGRAPHIC_AT = /\p{Ideographic}/yu;
+
+// RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
+const SUBDIVISION_FLAGS = ['gbeng', 'gbsct', 'gbwls'];
+
+/**
+ * @param {RegExp} re - sticky
+ * @param {string} text
+ * @param {number} index
+ */
+function matchesAt(re, text, index) {
+    re.lastIndex = index;
+    return re.test(text);
+}
+
+/**
+ * Index where the code point that ends just before `index` starts, or -1.
+ * @param {string} text
+ * @param {number} index
+ */
+function previousStart(text, index) {
+    if (index <= 0) return -1;
+    const low = text.charCodeAt(index - 1);
+    if (low >= 0xDC00 && low <= 0xDFFF && index >= 2) {
+        const high = text.charCodeAt(index - 2);
+        if (high >= 0xD800 && high <= 0xDBFF) return index - 2;
+    }
+    return index - 1;
+}
+
+/** @param {number} c */
+const isMongolianLetter = (c) =>
+    (c >= 0x1820 && c <= 0x1878) || (c >= 0x1880 && c <= 0x1884) || (c >= 0x1887 && c <= 0x18A8) || c === 0x18AA;
+
+/**
+ * A U+200D is benign between two Extended_Pictographic code points, allowing a
+ * skin-tone modifier or VS16 after the first.
+ * @param {string} text
+ * @param {number} i - index of the U+200D
+ */
+function isEmojiJoiner(text, i) {
+    let p = previousStart(text, i);
+    if (p < 0) return false;
+    const before = /** @type {number} */ (text.codePointAt(p));
+    if (before === 0xFE0F || (before >= 0x1F3FB && before <= 0x1F3FF)) p = previousStart(text, p);
+    return p >= 0 && matchesAt(PICTOGRAPHIC_AT, text, p) && matchesAt(PICTOGRAPHIC_AT, text, i + 1);
+}
+
+/**
+ * Length in UTF-16 units of the subdivision-flag tags starting at `i` (just after a
+ * U+1F3F4), or 0 when they are not exactly one of the RGI flags, as findHiddenRuns decides.
+ * @param {string} text
+ * @param {number} i
+ */
+function flagTagsAt(text, i) {
+    // five tag letters and CANCEL TAG, each a surrogate pair
+    if (text.charCodeAt(i + 10) !== 0xDB40 || text.charCodeAt(i + 11) !== 0xDC7F) return 0;
+    // more tags after CANCEL TAG make it one longer tag run, which is not a flag
+    const after = text.charCodeAt(i + 13);
+    if (text.charCodeAt(i + 12) === 0xDB40 && after >= 0xDC00 && after <= 0xDC7F) return 0;
+    let letters = '';
+    for (let k = 0; k < 10; k += 2) {
+        const low = text.charCodeAt(i + k + 1);
+        if (text.charCodeAt(i + k) !== 0xDB40 || low < 0xDC61 || low > 0xDC7A) return 0; // tag a-z
+        letters += String.fromCharCode(low - 0xDC00);
+    }
+    return SUBDIVISION_FLAGS.includes(letters) ? 12 : 0;
+}
+
+/**
+ * Locates invisible and control characters as inspectTextDetailed does, and marks
+ * the ones that belong to legitimate sequences as benign:
+ * - U+200D between two emoji (Extended_Pictographic), allowing a skin-tone modifier or
+ *   VS16 after the first: an emoji ZWJ sequence
+ * - U+FE0E or U+FE0F directly after an emoji, or between 0-9, # or * and U+20E3 (keycap)
+ * - one of U+E0100–U+E01EF directly after an ideograph: an ideographic variation sequence
+ * - U+180B–U+180D or U+180F directly after a Mongolian letter
+ * - the tags of the England, Scotland and Wales flags after U+1F3F4
+ * Nothing else is benign: not U+200B, U+2060, U+FEFF, bidi controls or other tags.
+ *
+ * `counts`, `total` and `truncated` are exactly those of inspectTextDetailed;
+ * `benignCounts` counts every benign character, listed or not. Two passes over the
+ * string, no copy; memory is flat with `maxFindings` set.
+ *
+ * @param {string} text
+ * @param {InspectDetailedOptions} [options]
+ * @returns {ContextReport}
+ */
+export function inspectTextInContext(text, options = {}) {
+    if (typeof text !== 'string') {
+        throw new TypeError('inspectTextInContext expects a string input.');
+    }
+    const report = inspectTextDetailed(text, options);
+    const findings = /** @type {ContextFinding[]} */ (report.findings);
+    for (const f of findings) f.benign = false;
+
+    /** @type {TrashCounts} */
+    const benignCounts = /** @type {TrashCounts} */ ({});
+    for (const key of Object.keys(report.counts)) benignCounts[/** @type {keyof TrashCounts} */ (key)] = 0;
+    let benignTotal = 0;
+    let next = 0; // first listed finding not yet passed
+
+    /**
+     * @param {number} offset
+     * @param {keyof TrashCounts} category
+     */
+    const markBenign = (offset, category) => {
+        benignCounts[category]++;
+        benignTotal++;
+        while (next < findings.length && findings[next].offset < offset) next++;
+        if (next < findings.length && findings[next].offset === offset) findings[next].benign = true;
+    };
+
+    for (let i = 0, n = text.length; i < n; i++) {
+        const c = text.charCodeAt(i);
+        if (c < 0x180B) continue; // the common case: nothing below can be benign
+
+        if (c === 0x200D) {
+            if (isEmojiJoiner(text, i)) markBenign(i, 'zero_width');
+        } else if (c === 0xFE0E || c === 0xFE0F) {
+            const p = previousStart(text, i);
+            const before = p < 0 ? -1 : text.charCodeAt(p);
+            const keycap = (before === 0x23 || before === 0x2A || (before >= 0x30 && before <= 0x39)) &&
+                text.charCodeAt(i + 1) === 0x20E3;
+            if (keycap || (p >= 0 && matchesAt(PICTOGRAPHIC_AT, text, p))) markBenign(i, 'variation_selector');
+        } else if ((c >= 0x180B && c <= 0x180D) || c === 0x180F) {
+            if (isMongolianLetter(text.charCodeAt(i - 1))) markBenign(i, 'variation_selector');
+        } else if (c === 0xD83C && text.charCodeAt(i + 1) === 0xDFF4) { // U+1F3F4
+            const length = flagTagsAt(text, i + 2);
+            for (let k = 0; k < length; k += 2) markBenign(i + 2 + k, 'tag');
+            i += 1 + length;
+        } else if (c === 0xDB40) { // plane 14: U+E0000-U+E03FF
+            const low = text.charCodeAt(i + 1);
+            if (low >= 0xDD00 && low <= 0xDDEF) { // U+E0100-U+E01EF
+                const p = previousStart(text, i);
+                if (p >= 0 && matchesAt(IDEOGRAPHIC_AT, text, p)) markBenign(i, 'variation_selector');
+            }
+            if (low >= 0xDC00 && low <= 0xDFFF) i++;
+        }
+    }
+
+    return { findings, counts: report.counts, total: report.total, truncated: report.truncated, benignCounts, benignTotal };
+}

package/src/hiddenText.js

@@ -40,6 +40,9 @@ export { VERSION } from './version.generated.js';
  * @property {Array<[number, number]>} benign - [offset, end) spans of legitimate
  *   sequences (subdivision flags, a single emoji or text presentation selector), at most `maxRuns` long
  * @property {HiddenRunCounts} counts - every run per kind, listed or not
+ * @property {HiddenRunCounts} codePointCounts - code points in those runs per kind, listed or not
+ * @property {number} supplementarySelectorCount - code points U+E0100–U+E01EF in those runs, which
+ *   (unlike the emoji selectors U+FE00–U+FE0F) can carry data; included in codePointCounts.variation_selector
  * @property {number} total - sum of `counts`
  * @property {number} benignTotal - every benign span, listed or not
  * @property {boolean} truncated - true when runs or benign spans were left out of the lists
@@ -164,6 +167,9 @@ export function findHiddenRuns(text, options = {}) {
     const benign = [];
     /** @type {HiddenRunCounts} */
     const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
+    /** @type {HiddenRunCounts} */
+    const codePointCounts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
+    let supplementarySelectorCount = 0;
     let total = 0;
     let benignTotal = 0;
     let line = 1;
@@ -184,12 +190,14 @@ export function findHiddenRuns(text, options = {}) {
 
         const offset = i;
         let count = 0;
+        let supplementary = 0;
         let last = cp;
         while (i < text.length) {
             const c = /** @type {number} */ (text.codePointAt(i));
             if (kindOf(c) !== kind) break;
             last = c;
             count++;
+            if (c >= 0xE0100 && c <= 0xE01EF) supplementary++;
             i += c > 0xFFFF ? 2 : 1;
         }
         const end = i;
@@ -207,6 +215,8 @@ export function findHiddenRuns(text, options = {}) {
         }
 
         counts[kind]++;
+        codePointCounts[kind] += count;
+        supplementarySelectorCount += supplementary;
         total++;
         if (runs.length < maxRuns) {
             /** @type {HiddenRun} */
@@ -219,5 +229,5 @@ export function findHiddenRuns(text, options = {}) {
             runs.push(run);
         }
     }
-    return { runs, benign, counts, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
+    return { runs, benign, counts, codePointCounts, supplementarySelectorCount, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
 }

package/src/version.generated.js

@@ -2,4 +2,4 @@
 // Generated from package.json by scripts/gen-version.js (`npm run gen`); do not edit.
 
 /** The installed version of text-sanctifier, as in its package.json. */
-export const VERSION = '1.3.6';
+export const VERSION = '1.3.7';