text-sanctifier 1.3.12 → 1.3.13

package/README.md

@@ -261,6 +261,7 @@ const { runs, benign, counts, total, truncated } = findHiddenRuns(text, { maxRun
 
 - `kind` is `tag` (U+E0000–U+E007F), `variation_selector` (U+FE00–U+FE0F, U+E0100–U+E01EF, U+180B–U+180D, U+180F) `invisible_operator` (U+2061–U+2064) or `reserved_tag_plane` (U+E0080–U+E00FF, U+E01F0–U+E0FFF: unassigned, default ignorable, so they render as nothing and can carry data). Consecutive characters of one kind form one run.
 - `offset` and `end` are UTF-16 indexes, `[offset, end)`; `count` is in code points. `line` and `column` are counted exactly as in `inspectTextDetailed` (1-based, columns in UTF-16 units, CRLF is one line break). Only tag runs have `decoded` and `decodedTruncated`.
+- `sequence` marks a lone selector in a form `text-sanctifier/context` calls benign: `'ideographic'` for U+E0100–U+E01EF after an ideograph, `'mongolian'` for U+180B–U+180D or U+180F after a Mongolian letter. These stay in `runs` because they can carry data, up to a byte per ideograph, so a run with `sequence` is usually ordinary text, but many of them can be a message. U+FE0E and U+FE0F have no such room and go under `benign`.
 - `maxRuns` caps the `runs` list and, separately, the `benign` list; `counts`, `total` and `benignTotal` count everything, and `truncated` is true when either list was cut. `maxDecodedLength` caps each `decoded` string in UTF-16 units and never cuts a `<U+XXXX>` marker. Both default to no cap. Only non-negative integers and `Infinity` are valid; invalid values (negative, fractional, a string, `NaN` or `null`) also mean no cap, so set them explicitly for untrusted input. With both set, memory stays flat on any input: on 16 M characters (32 MB) of hidden text the heap grows by under 1 MB, against hundreds of MB with no caps.
 - `decodeTagText` turns U+E0020–U+E007E into the ASCII they shadow and writes any other character as `<U+XXXX>`.
 - Zero-width, bidi and other invisible characters are not runs; use `inspectTextDetailed` for those.
@@ -365,7 +366,7 @@ The repository publishes a [pre-commit](https://pre-commit.com) hook manifest, s
 # .pre-commit-config.yaml
 repos:
   - repo: https://github.com/iWhatty/text-sanctifier
-    rev: v1.3.12  # a release tag
+    rev: v1.3.13  # a release tag
     hooks:
       - id: text-sanctifier            # every staged text file
       - id: text-sanctifier-ai-config  # agent instruction files, with --ai-config

package/dist/text-sanctifier.min.js

@@ -3,4 +3,4 @@ var y=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u2
 `){return e.replace(re,r)}var te=/[ \t]*(\n+)[ \t]*|([ \t]+)/g;function ie(e){return e.replace(te,"$1$2")}var oe=/\n{2,}/g,ue=/\n{3,}/g;function se(e,r){return r?e.replace(ue,`
 
 `):e.replace(oe,`
-`)}var le=/ {2,}/g;function ae(e){return e.replace(le," ")}function fe(e){return e.replace(y,"")}function ce(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of A(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var Ee="1.3.12";export{Ee as VERSION,R as getNewlineStats,H as getRecommendedSanctifierOptions,j as inspectText,A as inspectTextDetailed,ce as revealInvisible,L as sanctifyKeyboardOnly,T as sanctifyKeyboardOnlyEmoji,N as sanctifyLoose,D as sanctifyPasted,w as sanctifyStrict,J as summonSanctifier};
+`)}var le=/ {2,}/g;function ae(e){return e.replace(le," ")}function fe(e){return e.replace(y,"")}function ce(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of A(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var Ee="1.3.13";export{Ee as VERSION,R as getNewlineStats,H as getRecommendedSanctifierOptions,j as inspectText,A as inspectTextDetailed,ce as revealInvisible,L as sanctifyKeyboardOnly,T as sanctifyKeyboardOnlyEmoji,N as sanctifyLoose,D as sanctifyPasted,w as sanctifyStrict,J as summonSanctifier};

package/dist/types/hiddenScan.d.ts

@@ -25,6 +25,7 @@ export function scanHiddenSpans(text: string, options?: HiddenScanOptions, onSpa
 }) => void): HiddenTextScan;
 export type HiddenRunKind = import("./hiddenText.js").HiddenRunKind;
 export type HiddenRun = import("./hiddenText.js").HiddenRun;
+export type HiddenRunSequence = import("./hiddenText.js").HiddenRunSequence;
 export type HiddenRunCounts = import("./hiddenText.js").HiddenRunCounts;
 export type HiddenTextScan = import("./hiddenText.js").HiddenTextScan;
 export type HiddenScanOptions = import("./hiddenText.js").HiddenScanOptions;

package/dist/types/hiddenText.d.ts

@@ -1,4 +1,5 @@
 /** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
+/** @typedef {'ideographic' | 'mongolian'} HiddenRunSequence */
 /**
  * @typedef {Object} HiddenRun
  * @property {HiddenRunKind} kind
@@ -10,6 +11,9 @@
  * @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText),
  *   at most `maxDecodedLength` UTF-16 units
  * @property {boolean} [decodedTruncated] - tag runs only: true when `decoded` stops short of the run
+ * @property {HiddenRunSequence} [sequence] - a lone selector in a form text-sanctifier/context calls
+ *   benign: U+E0100–U+E01EF after an ideograph, or U+180B–U+180D or U+180F after a Mongolian letter.
+ *   Still a run, because these can carry data: up to a byte per ideograph.
  */
 /**
  * @typedef {Object} HiddenRunCounts
@@ -57,6 +61,7 @@ export function decodeTagText(text: string): string;
  *    Extended_Pictographic character, or between #, * or a digit and U+20E3 (a keycap),
  *    the same rule as text-sanctifier/context. After a letter it is a run: one selector
  *    per letter can carry a bit each.
+ * Lone ideographic and Mongolian selectors stay runs, marked with `sequence`.
  *
  * `line` and `column` are counted as in inspectTextDetailed. Every run and benign
  * span is counted; only the first `maxRuns` of each are listed. With the caps set,
@@ -70,6 +75,7 @@ export function decodeTagText(text: string): string;
 export function findHiddenRuns(text: string, options?: HiddenScanOptions): HiddenTextScan;
 export { VERSION } from "./version.generated.js";
 export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator" | "reserved_tag_plane";
+export type HiddenRunSequence = "ideographic" | "mongolian";
 export type HiddenRun = {
     kind: HiddenRunKind;
     /**
@@ -101,6 +107,12 @@ export type HiddenRun = {
      * - tag runs only: true when `decoded` stops short of the run
      */
     decodedTruncated?: boolean;
+    /**
+     * - a lone selector in a form text-sanctifier/context calls
+     * benign: U+E0100–U+E01EF after an ideograph, or U+180B–U+180D or U+180F after a Mongolian letter.
+     * Still a run, because these can carry data: up to a byte per ideograph.
+     */
+    sequence?: HiddenRunSequence;
 };
 export type HiddenRunCounts = {
     tag: number;

package/dist/types/version.generated.d.ts

@@ -1,2 +1,2 @@
 /** The installed version of text-sanctifier, as in its package.json. */
-export const VERSION: "1.3.12";
+export const VERSION: "1.3.13";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "text-sanctifier",
-  "version": "1.3.12",
+  "version": "1.3.13",
   "type": "module",
   "description": "Find and remove invisible Unicode in web input, model-bound text and repositories.",
   "engines": {

package/src/hiddenScan.js

@@ -3,6 +3,7 @@
 /**
  * @typedef {import('./hiddenText.js').HiddenRunKind} HiddenRunKind
  * @typedef {import('./hiddenText.js').HiddenRun} HiddenRun
+ * @typedef {import('./hiddenText.js').HiddenRunSequence} HiddenRunSequence
  * @typedef {import('./hiddenText.js').HiddenRunCounts} HiddenRunCounts
  * @typedef {import('./hiddenText.js').HiddenTextScan} HiddenTextScan
  * @typedef {import('./hiddenText.js').HiddenScanOptions} HiddenScanOptions
@@ -76,6 +77,32 @@ function isPresentationBase(text, index) {
     return PICTOGRAPHIC_AT.test(text);
 }
 
+const IDEOGRAPHIC_AT = /\p{Ideographic}/yu;
+
+/**
+ * The legitimate sequence a lone selector at `index` could belong to, by the same rules
+ * as text-sanctifier/context. It stays a run: unlike U+FE0E and U+FE0F, these selectors
+ * have enough values to spell data, up to a byte per ideograph.
+ * @param {string} text
+ * @param {number} index
+ * @param {number} cp - the selector
+ * @returns {HiddenRunSequence | null}
+ */
+function sequenceOf(text, index, cp) {
+    if (cp >= 0xE0100 && cp <= 0xE01EF) {
+        if (index === 0) return null;
+        const low = text.charCodeAt(index - 1);
+        IDEOGRAPHIC_AT.lastIndex = low >= 0xDC00 && low <= 0xDFFF && index >= 2 &&
+            (text.charCodeAt(index - 2) & 0xFC00) === 0xD800 ? index - 2 : index - 1;
+        return IDEOGRAPHIC_AT.test(text) ? 'ideographic' : null;
+    }
+    if (cp > 0x180F) return null;
+    const c = text.charCodeAt(index - 1);
+    return (c >= 0x1820 && c <= 0x1878) || (c >= 0x1880 && c <= 0x1884) || (c >= 0x1887 && c <= 0x18A8) || c === 0x18AA
+        ? 'mongolian'
+        : null;
+}
+
 /**
  * A cap option: a non-negative integer or Infinity; anything else means no cap.
  * @param {unknown} value
@@ -163,6 +190,9 @@ export function scanHiddenSpans(text, options = {}, onSpan) {
                 const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
                 run.decoded = decoded;
                 run.decodedTruncated = truncated;
+            } else if (kind === 'variation_selector' && count === 1) {
+                const sequence = sequenceOf(text, offset, cp);
+                if (sequence) run.sequence = sequence;
             }
             runs.push(run);
         }

package/src/hiddenText.js

@@ -15,6 +15,8 @@ export { VERSION } from './version.generated.js';
 
 /** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
 
+/** @typedef {'ideographic' | 'mongolian'} HiddenRunSequence */
+
 /**
  * @typedef {Object} HiddenRun
  * @property {HiddenRunKind} kind
@@ -26,6 +28,9 @@ export { VERSION } from './version.generated.js';
  * @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText),
  *   at most `maxDecodedLength` UTF-16 units
  * @property {boolean} [decodedTruncated] - tag runs only: true when `decoded` stops short of the run
+ * @property {HiddenRunSequence} [sequence] - a lone selector in a form text-sanctifier/context calls
+ *   benign: U+E0100–U+E01EF after an ideograph, or U+180B–U+180D or U+180F after a Mongolian letter.
+ *   Still a run, because these can carry data: up to a byte per ideograph.
  */
 
 /**
@@ -81,6 +86,7 @@ export function decodeTagText(text) {
  *    Extended_Pictographic character, or between #, * or a digit and U+20E3 (a keycap),
  *    the same rule as text-sanctifier/context. After a letter it is a run: one selector
  *    per letter can carry a bit each.
+ * Lone ideographic and Mongolian selectors stay runs, marked with `sequence`.
  *
  * `line` and `column` are counted as in inspectTextDetailed. Every run and benign
  * span is counted; only the first `maxRuns` of each are listed. With the caps set,

package/src/version.generated.js

@@ -2,4 +2,4 @@
 // Generated from package.json by scripts/gen-version.js (`npm run gen`); do not edit.
 
 /** The installed version of text-sanctifier, as in its package.json. */
-export const VERSION = '1.3.12';
+export const VERSION = '1.3.13';