text-sanctifier 1.3.4 → 1.3.6

package/src/hiddenText.js

@@ -1,14 +1,17 @@
 // src/hiddenText.js
 //
 // Finds runs of the characters used to hide instructions in text a model reads:
-// tag characters, variation selectors and invisible operators. Tag runs are decoded
+// tag characters, variation selectors, invisible operators and the reserved code points
+// of the tag plane. Tag runs are decoded
 // to the ASCII they spell. Used by `text-sanctifier check --ai-config`, and exported
 // as `text-sanctifier/hidden`.
 //
 // Plain JavaScript with no Node APIs, so it also runs in a browser. src/index.js does
 // not import this file, so it stays out of the browser bundle and its size budget.
 
-/** @typedef {'tag' | 'variation_selector' | 'invisible_operator'} HiddenRunKind */
+export { VERSION } from './version.generated.js';
+
+/** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
 
 /**
  * @typedef {Object} HiddenRun
@@ -16,14 +19,38 @@
  * @property {number} offset - UTF-16 index of the first character
  * @property {number} end - UTF-16 index just past the last character
  * @property {number} count - number of code points in the run
- * @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText)
+ * @property {number} line - 1-based line of the first character (as in inspectTextDetailed)
+ * @property {number} column - 1-based column of the first character, in UTF-16 units
+ * @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText),
+ *   at most `maxDecodedLength` UTF-16 units
+ * @property {boolean} [decodedTruncated] - tag runs only: true when `decoded` stops short of the run
+ */
+
+/**
+ * @typedef {Object} HiddenRunCounts
+ * @property {number} tag
+ * @property {number} variation_selector
+ * @property {number} invisible_operator
+ * @property {number} reserved_tag_plane
  */
 
 /**
  * @typedef {Object} HiddenTextScan
- * @property {HiddenRun[]} runs - runs that may hide text, in order
+ * @property {HiddenRun[]} runs - runs that may hide text, in order, at most `maxRuns` long
  * @property {Array<[number, number]>} benign - [offset, end) spans of legitimate
- *   sequences (subdivision flags, a single emoji or text presentation selector)
+ *   sequences (subdivision flags, a single emoji or text presentation selector), at most `maxRuns` long
+ * @property {HiddenRunCounts} counts - every run per kind, listed or not
+ * @property {number} total - sum of `counts`
+ * @property {number} benignTotal - every benign span, listed or not
+ * @property {boolean} truncated - true when runs or benign spans were left out of the lists
+ */
+
+/**
+ * @typedef {Object} HiddenScanOptions
+ * @property {number} [maxRuns=Infinity] - cap on listed runs, and separately on listed benign spans
+ *   (non-negative integer or Infinity). Counts stay exact.
+ * @property {number} [maxDecodedLength=Infinity] - cap on each run's `decoded` text, in UTF-16 units.
+ *   For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
  */
 
 const TAG_FIRST = 0xE0000;
@@ -43,6 +70,8 @@ function kindOf(cp) {
     if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
         (cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
     if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
+    // Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
+    if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
     return null;
 }
 
@@ -57,35 +86,101 @@ const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
  * @returns {string}
  */
 export function decodeTagText(text) {
+    return decodeRange(text, 0, text.length, Infinity).decoded;
+}
+
+/**
+ * Decodes text[start, end) as decodeTagText does, stopping before the output would
+ * pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
+ * @param {string} text
+ * @param {number} start
+ * @param {number} end
+ * @param {number} limit
+ * @returns {{ decoded: string, truncated: boolean }}
+ */
+function decodeRange(text, start, end, limit) {
     let out = '';
-    for (const ch of text) {
-        const cp = /** @type {number} */ (ch.codePointAt(0));
-        out += cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
+    let i = start;
+    while (i < end) {
+        const cp = /** @type {number} */ (text.codePointAt(i));
+        const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
+        if (out.length + piece.length > limit) return { decoded: out, truncated: true };
+        out += piece;
+        i += cp > 0xFFFF ? 2 : 1;
     }
-    return out;
+    return { decoded: out, truncated: false };
 }
 
+// Whitespace as JavaScript's \s defines it, tested at an index of the text itself,
+// so the check allocates nothing.
+const SPACE_AT = /\s/y;
+
 /**
- * Finds runs of tag characters, variation selectors and invisible operators.
+ * @param {string} text
+ * @param {number} index
+ * @returns {boolean}
+ */
+function isSpaceAt(text, index) {
+    SPACE_AT.lastIndex = index;
+    return SPACE_AT.test(text);
+}
+
+/**
+ * A cap option: a non-negative integer or Infinity; anything else means no cap.
+ * @param {unknown} value
+ * @returns {number}
+ */
+function cap(value) {
+    return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
+        ? value
+        : Infinity;
+}
+
+/**
+ * Finds runs of tag characters, variation selectors, invisible operators and reserved
+ * tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
  * Consecutive characters of the same kind form one run. Two kinds of sequence are
  * legitimate and listed under `benign` instead of `runs`:
  *  - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
  *    tags for "gbeng", "gbsct" or "gbwls" and U+E007F
  *  - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
+ *
+ * `line` and `column` are counted as in inspectTextDetailed. Every run and benign
+ * span is counted; only the first `maxRuns` of each are listed. With the caps set,
+ * memory stays flat however large the input is. One pass over the string, no copy.
+ * Set both caps for untrusted input: without them, results grow with the input.
+ *
  * @param {string} text
+ * @param {HiddenScanOptions} [options]
  * @returns {HiddenTextScan}
  */
-export function findHiddenRuns(text) {
+export function findHiddenRuns(text, options = {}) {
+    const maxRuns = cap(options && options.maxRuns);
+    const maxDecodedLength = cap(options && options.maxDecodedLength);
+
     /** @type {HiddenRun[]} */
     const runs = [];
     /** @type {Array<[number, number]>} */
     const benign = [];
+    /** @type {HiddenRunCounts} */
+    const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
+    let total = 0;
+    let benignTotal = 0;
+    let line = 1;
+    let lineStart = 0;
 
     let i = 0;
     while (i < text.length) {
         const cp = /** @type {number} */ (text.codePointAt(i));
         const kind = kindOf(cp);
-        if (kind === null) { i += cp > 0xFFFF ? 2 : 1; continue; }
+        if (kind === null) {
+            if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
+                line++;
+                lineStart = i + 1;
+            }
+            i += cp > 0xFFFF ? 2 : 1;
+            continue;
+        }
 
         const offset = i;
         let count = 0;
@@ -99,22 +194,30 @@ export function findHiddenRuns(text) {
         }
         const end = i;
 
-        if (kind === 'tag') {
-            const body = text.slice(offset, end);
-            const afterFlag = offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG;
-            if (afterFlag && last === CANCEL_TAG && SUBDIVISION_FLAGS.has(decodeTagText(body).slice(0, -'<U+E007F>'.length))) {
-                benign.push([offset, end]);
-            } else {
-                runs.push({ kind, offset, end, count, decoded: decodeTagText(body) });
-            }
+        // A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
+        const isBenign = kind === 'tag'
+            ? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
+              SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
+            : kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
+              offset > 0 && !isSpaceAt(text, offset - 1);
+        if (isBenign) {
+            benignTotal++;
+            if (benign.length < maxRuns) benign.push([offset, end]);
             continue;
         }
-        if (kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
-            offset > 0 && !/\s/.test(text[offset - 1])) {
-            benign.push([offset, end]);
-            continue;
+
+        counts[kind]++;
+        total++;
+        if (runs.length < maxRuns) {
+            /** @type {HiddenRun} */
+            const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
+            if (kind === 'tag') {
+                const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
+                run.decoded = decoded;
+                run.decodedTruncated = truncated;
+            }
+            runs.push(run);
         }
-        runs.push({ kind, offset, end, count });
     }
-    return { runs, benign };
+    return { runs, benign, counts, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
 }

package/src/index.d.ts

@@ -3,7 +3,14 @@ import { inspectTextDetailed } from './inspectText.js';
 import { getNewlineStats } from './inspectText.js';
 import { getRecommendedSanctifierOptions } from './inspectText.js';
 import { summonSanctifier } from './sanctifyText.js';
-export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier };
+import { sanctifyStrict } from './sanctifyText.js';
+import { sanctifyLoose } from './sanctifyText.js';
+import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
+import { sanctifyKeyboardOnly } from './sanctifyText.js';
+import { sanctifyPasted } from './sanctifyText.js';
+import { revealInvisible } from './reveal.js';
+import { VERSION } from './version.generated.js';
+export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
 export type UnicodeTrashReport = import('./inspectText.js').UnicodeTrashReport;
 export type TrashCategory = import('./inspectText.js').TrashCategory;
 export type TrashFinding = import('./inspectText.js').TrashFinding;
@@ -12,3 +19,4 @@ export type DetailedTrashReport = import('./inspectText.js').DetailedTrashReport
 export type InspectDetailedOptions = import('./inspectText.js').InspectDetailedOptions;
 export type NewlineStats = import('./inspectText.js').NewlineStats;
 export type SanctifyOptions = import('./sanctifyText.js').SanctifyOptions;
+export type RevealOptions = import('./reveal.js').RevealOptions;

package/src/index.js

@@ -4,8 +4,14 @@
 import { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions } from './inspectText.js';
 export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions };
 
-import { summonSanctifier } from './sanctifyText.js';
-export { summonSanctifier };
+import { summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted } from './sanctifyText.js';
+export { summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted };
+
+import { revealInvisible } from './reveal.js';
+export { revealInvisible };
+
+import { VERSION } from './version.generated.js';
+export { VERSION };
 
 // Type-only re-exports so TypeScript consumers can name these from the package root.
 /** @typedef {import('./inspectText.js').UnicodeTrashReport} UnicodeTrashReport */
@@ -16,6 +22,7 @@ export { summonSanctifier };
 /** @typedef {import('./inspectText.js').InspectDetailedOptions} InspectDetailedOptions */
 /** @typedef {import('./inspectText.js').NewlineStats} NewlineStats */
 /** @typedef {import('./sanctifyText.js').SanctifyOptions} SanctifyOptions */
+/** @typedef {import('./reveal.js').RevealOptions} RevealOptions */
 
 
 // In Future: export helpers for power users

package/src/inspectText.js

@@ -1,13 +1,7 @@
 // ./src/inspectText.js
 
-import {
-    CONTROL_CHARS_REGEX,
-    INVISIBLE_TRASH_REGEX,
-    EMOJI_REGEX,
-    ASCII_KEYBOARD_SAFE_REGEX,
-    normalizeTypographicJank,
-} from './sanctifyText.js';
-import { classifyCodeUnit, classifyAstral } from './categories.generated.js';
+import { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX, classifyCodeUnit, classifyAstral } from './categories.generated.js';
+import { EMOJI_REGEX, ASCII_KEYBOARD_SAFE_REGEX, normalizeTypographicJank } from './textPatterns.js';
 
 /** @typedef {import('./sanctifyText.js').SanctifyOptions} SanctifyOptions */
 

package/src/reveal.js

@@ -0,0 +1,40 @@
+// ./src/reveal.js
+
+import { inspectTextDetailed } from './inspectText.js';
+
+/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
+
+/**
+ * @typedef {Object} RevealOptions
+ * @property {(codePoint: number, category: TrashCategory) => string} [format] - Text shown in place of
+ *   each hidden character. Default: `<U+XXXX>` (at least four uppercase hex digits).
+ */
+
+/**
+ * Makes hidden characters visible: every character `inspectTextDetailed` would report is
+ * replaced by a marker such as `<U+200B>`, and everything else is kept as it is,
+ * including tabs, newlines and emoji. Use it to show hidden text in logs, diffs, review
+ * tools and agent tool output. A literal `<U+200B>` already in the text looks the same as
+ * a marker; pass `format` to choose markers that can't occur in your input.
+ *
+ * @param {string} text
+ * @param {RevealOptions} [options]
+ * @returns {string}
+ */
+export function revealInvisible(text, options = {}) {
+    if (typeof text !== 'string') {
+        throw new TypeError('revealInvisible expects a string input.');
+    }
+    const format = options && typeof options.format === 'function'
+        ? options.format
+        : (codePoint) => '<U+' + codePoint.toString(16).toUpperCase().padStart(4, '0') + '>';
+
+    // One finding per hidden character, in offset order; code points above U+FFFF span two units
+    let out = '';
+    let last = 0;
+    for (const f of inspectTextDetailed(text, { maxFindings: Infinity }).findings) {
+        out += text.slice(last, f.offset) + format(f.codePoint, f.category);
+        last = f.offset + (f.codePoint > 0xFFFF ? 2 : 1);
+    }
+    return out + text.slice(last);
+}