text-sanctifier 1.3.5 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -7
- package/bin/text-sanctifier.js +1 -1
- package/dist/text-sanctifier.min.js +5 -5
- package/dist/types/hiddenText.d.ts +62 -6
- package/dist/types/index.d.ts +7 -1
- package/dist/types/sanctifyText.d.ts +141 -110
- package/dist/types/textPatterns.d.ts +15 -0
- package/dist/types/version.generated.d.ts +2 -0
- package/package.json +4 -4
- package/src/hiddenText.js +129 -26
- package/src/index.d.ts +7 -1
- package/src/index.js +5 -2
- package/src/inspectText.js +2 -8
- package/src/sanctifyText.js +138 -141
- package/src/textPatterns.js +78 -0
- package/src/version.generated.js +5 -0
package/README.md
CHANGED
|
@@ -119,6 +119,16 @@ Two options handle spaces and zero-width characters gently, and both run before
|
|
|
119
119
|
- `normalizeSpaces`: replace look-alike spaces with an ordinary space. These are U+00A0, U+2000–U+200A, U+202F and U+205F: the `nonbreaking_space` category except U+3000, the ordinary space of CJK text. With `purgeInvisibleChars` alone, a non-breaking space between two words is deleted and the words run together; with `normalizeSpaces` as well, they stay apart.
|
|
120
120
|
- `purgeZeroWidthSpaces`: remove the zero width space U+200B, the word joiner U+2060 and U+FEFF, and nothing else. Emoji joiners, ZWNJ, variation selectors and bidi marks are kept.
|
|
121
121
|
|
|
122
|
+
### Presets as standalone imports
|
|
123
|
+
|
|
124
|
+
Each preset is also its own export: `sanctifyStrict`, `sanctifyLoose`, `sanctifyKeyboardOnly`, `sanctifyKeyboardOnlyEmoji` and `sanctifyPasted`. They are the same functions as `summonSanctifier.strict` and the rest. Import one directly and a bundler leaves out `summonSanctifier` and the other presets (about 1.1 KB gzipped instead of 1.45 KB).
|
|
125
|
+
|
|
126
|
+
```js
|
|
127
|
+
import { sanctifyPasted } from 'text-sanctifier';
|
|
128
|
+
|
|
129
|
+
const output = sanctifyPasted(clipboardText);
|
|
130
|
+
```
|
|
131
|
+
|
|
122
132
|
### `summonSanctifier.strict`
|
|
123
133
|
|
|
124
134
|
Aggressively purges: emojis, control characters, extra spacing, and newlines.
|
|
@@ -236,22 +246,25 @@ getNewlineStats('a\r\nb\n'); // { crlf: 1, cr: 0, lf: 1, types: ['CRLF', 'LF'],
|
|
|
236
246
|
|
|
237
247
|
The categories cover exactly what `nukeControls` and `purgeInvisibleChars` strip. Every code point with the Unicode property Default_Ignorable_Code_Point (characters renderers must draw as nothing) is in a category, and a test checks this against the running Node's Unicode data.
|
|
238
248
|
|
|
239
|
-
### `text-sanctifier/hidden`: `findHiddenRuns(text)` and `decodeTagText(text)`
|
|
249
|
+
### `text-sanctifier/hidden`: `findHiddenRuns(text, options?)` and `decodeTagText(text)`
|
|
240
250
|
|
|
241
251
|
The run finder behind [`check --ai-config`](#ai-configuration-files), for tools that read text on its way to a model. It has no Node APIs, so it works in a browser too, but it is a separate entry point and not part of the `text-sanctifier/browser` bundle.
|
|
242
252
|
|
|
243
253
|
```js
|
|
244
254
|
import { findHiddenRuns, decodeTagText } from 'text-sanctifier/hidden';
|
|
245
255
|
|
|
246
|
-
const { runs, benign } = findHiddenRuns(text);
|
|
247
|
-
// runs: [{ kind: 'tag', offset, end, count, decoded: 'Ignore the user' }, ...]
|
|
256
|
+
const { runs, benign, counts, total, truncated } = findHiddenRuns(text, { maxRuns: 100, maxDecodedLength: 1000 });
|
|
257
|
+
// runs: [{ kind: 'tag', offset, end, line, column, count, decoded: 'Ignore the user', decodedTruncated: false }, ...]
|
|
248
258
|
// benign: [[offset, end], ...] England/Scotland/Wales flags, a lone U+FE0E/U+FE0F after a character
|
|
259
|
+
// counts: { tag, variation_selector, invisible_operator, reserved_tag_plane }, every run listed or not; total is their sum
|
|
249
260
|
```
|
|
250
261
|
|
|
251
|
-
- `kind` is `tag` (U+E0000–U+E007F), `variation_selector` (U+FE00–U+FE0F, U+E0100–U+E01EF, U+180B–U+180D, U+180F)
|
|
252
|
-
- `offset` and `end` are UTF-16 indexes, `[offset, end)`; `count` is in code points. Only tag runs have `decoded`.
|
|
262
|
+
- `kind` is `tag` (U+E0000–U+E007F), `variation_selector` (U+FE00–U+FE0F, U+E0100–U+E01EF, U+180B–U+180D, U+180F) `invisible_operator` (U+2061–U+2064) or `reserved_tag_plane` (U+E0080–U+E00FF, U+E01F0–U+E0FFF: unassigned, default ignorable, so they render as nothing and can carry data). Consecutive characters of one kind form one run.
|
|
263
|
+
- `offset` and `end` are UTF-16 indexes, `[offset, end)`; `count` is in code points. `line` and `column` are counted exactly as in `inspectTextDetailed` (1-based, columns in UTF-16 units, CRLF is one line break). Only tag runs have `decoded` and `decodedTruncated`.
|
|
264
|
+
- `maxRuns` caps the `runs` list and, separately, the `benign` list; `counts`, `total` and `benignTotal` count everything, and `truncated` is true when either list was cut. `maxDecodedLength` caps each `decoded` string in UTF-16 units and never cuts a `<U+XXXX>` marker. Both default to no cap, and an invalid value (negative, fractional, a string) also means no cap, so set them explicitly for untrusted input. With both set, memory stays flat on any input: on 16 M characters (32 MB) of hidden text the heap grows by under 1 MB, against hundreds of MB with no caps.
|
|
253
265
|
- `decodeTagText` turns U+E0020–U+E007E into the ASCII they shadow and writes any other character as `<U+XXXX>`.
|
|
254
266
|
- Zero-width, bidi and other invisible characters are not runs; use `inspectTextDetailed` for those.
|
|
267
|
+
- `VERSION` is exported here and from the package root: the installed version, as in `package.json`.
|
|
255
268
|
|
|
256
269
|
---
|
|
257
270
|
|
|
@@ -301,7 +314,7 @@ skills/review/SKILL.md:2:3 U+FE00 run of 3 variation selectors
|
|
|
301
314
|
- Consecutive tag characters, variation selectors or invisible operators are reported once, as a run, at the first character. Tag runs are decoded: U+E0020–U+E007E become the ASCII they shadow, and other tags are shown as `<U+XXXX>`.
|
|
302
315
|
- Every other invisible or control character is reported as in a plain `check`.
|
|
303
316
|
- Two legitimate sequences are left out: the flags of England, Scotland and Wales (U+1F3F4, tag letters, U+E007F), and a single U+FE0E or U+FE0F after another character (text or emoji presentation). A plain `check` still reports both.
|
|
304
|
-
- `--json` adds `hiddenRuns` (`{ kind, offset, line, column, codePoint, count, decoded?, decodedTruncated? }`, at most `--max-findings` per file) and `hiddenRunCount` to each file, and `hiddenRunCount` to the summary. `kind` is `tag`, `variation_selector` or `
|
|
317
|
+
- `--json` adds `hiddenRuns` (`{ kind, offset, line, column, codePoint, count, decoded?, decodedTruncated? }`, at most `--max-findings` per file) and `hiddenRunCount` to each file, and `hiddenRunCount` to the summary. `kind` is `tag`, `variation_selector`, `invisible_operator` or `reserved_tag_plane`; `decoded` is cut at 1000 characters. `total` and `counts` leave out the legitimate sequences.
|
|
305
318
|
|
|
306
319
|
### SARIF output
|
|
307
320
|
|
|
@@ -321,7 +334,7 @@ The repository publishes a [pre-commit](https://pre-commit.com) hook manifest, s
|
|
|
321
334
|
# .pre-commit-config.yaml
|
|
322
335
|
repos:
|
|
323
336
|
- repo: https://github.com/iWhatty/text-sanctifier
|
|
324
|
-
rev: v1.3.
|
|
337
|
+
rev: v1.3.6 # a release tag
|
|
325
338
|
hooks:
|
|
326
339
|
- id: text-sanctifier # every staged text file
|
|
327
340
|
- id: text-sanctifier-ai-config # agent instruction files, with --ai-config
|
package/bin/text-sanctifier.js
CHANGED
|
@@ -443,7 +443,7 @@ const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
|
443
443
|
// --- check --ai-config ---
|
|
444
444
|
|
|
445
445
|
const MAX_DECODED = 1000;
|
|
446
|
-
const RUN_NOUNS = { tag: 'tag character', variation_selector: 'variation selector', invisible_operator: 'invisible operator' };
|
|
446
|
+
const RUN_NOUNS = { tag: 'tag character', variation_selector: 'variation selector', invisible_operator: 'invisible operator', reserved_tag_plane: 'reserved plane-14 code point' };
|
|
447
447
|
|
|
448
448
|
/**
|
|
449
449
|
* inspect() for agent instruction files: drops findings that belong to legitimate
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
var
|
|
2
|
-
|
|
1
|
+
var y=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g,d=/[\u00A0\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180B-\u180F\u2000-\u200F\u202A-\u202F\u205F-\u206F\u3000\u3164\uFE00-\uFE0F\uFEFF\uFFA0\uFFF0-\uFFF8\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0000}-\u{E0FFF}]+/gu;function A(e){return e<8192?D(e):v(e)}function D(e){return e<174?e<32?e<11?e<1?"nul":e<9?"c0_control":null:e<13?"c0_control":e<14?null:"c0_control":e<160?e<127?null:e<128?"c0_control":"c1_control":e<161?"nonbreaking_space":e<173?null:"invisible_format":e<4449?e<1564?e<847?null:e<848?"invisible_format":null:e<1565?"bidi_control":e<4447?null:"hangul_filler":e<6155?e<6068?null:e<6070?"invisible_format":null:e<6159?e<6158?"variation_selector":"invisible_format":e<6160?"variation_selector":null}function v(e){return e<8304?e<8240?e<8208?e<8203?"nonbreaking_space":e<8206?"zero_width":"bidi_control":e<8234?null:e<8239?"bidi_control":"nonbreaking_space":e<8289?e<8287?null:e<8288?"nonbreaking_space":"zero_width":e<8294?"invisible_format":e<8298?"bidi_control":"invisible_format":e<65040?e<12644?e<12288?null:e<12289?"nonbreaking_space":null:e<12645?"hangul_filler":e<65024?null:"variation_selector":e<65440?e<65279?null:e<65280?"bom":null:e<65520?e<65441?"hangul_filler":null:e<65529?"invisible_format":null}function I(e){return e<917504?e<113828?e<113824?null:"invisible_format":e<119155?null:e<119163?"invisible_format":null:e<917760?e<917632?"tag":"invisible_format":e<918e3?"variation_selector":e<921600?"invisible_format":null}var g=/[^\x20-\x7E\r\n]+/gu,B=/[\u2018\u2019\u201A\u201B\u2032\u2035]/g,G=/[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g,X=/[\u2012\u2013\u2014\u2015\u2212]/g,k=/\u2026/g,O=/[\u2022\u00B7]/g,P=/[\uFF01-\uFF5E]/g;function S(e){return e.replace(B,"'").replace(G,'"').replace(X,"-").replace(k,"...").replace(O,"*").replace(P,r=>String.fromCharCode(r.charCodeAt(0)-65248))}function M(){try{return new RegExp("(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)","gu")}catch{return/[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu}}var x=M();function m(e,r){return e.lastIndex=0,e.test(r)}function U(e){let r=S(e);if(!m(g,r))return!1;for(let n of r)if(!(n===`
|
|
2
|
+
`||n==="\r"||n>=" "&&n<="~")&&!m(x,n))return!0;return!1}function j(e){if(typeof e!="string")throw new TypeError("inspectText expects a string input.");let r=[],n={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:r},u=(s,c,f)=>{s&&(n[c]=!0,r.push(f))};u(m(y,e),"hasControlChars","Control characters detected."),u(m(d,e),"hasInvisibleChars","Invisible Unicode characters detected."),u(m(x,e),"hasEmojis","Emojis detected.");let{mixed:o,types:i}=b(e);return n.hasMixedNewlines=o,n.newlineStyle=o?"Mixed":i[0]||null,n.newlineStyle&&r.push(o?"Mixed newline styles detected.":`Consistent newline style: ${n.newlineStyle}`),u(U(e),"hasNonKeyboardChars","Non-keyboard characters detected."),n}var z=100;function C(e,r={}){if(typeof e!="string")throw new TypeError("inspectTextDetailed expects a string input.");let n=r&&r.maxFindings,u=n===1/0||Number.isInteger(n)&&n>=0?n:z,o={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},i=[],s=0,c=1,f=0;for(let a=0,h=e.length;a<h;a++){let l=e.charCodeAt(a);if(l>=32&&l<127)continue;if(l===10){c++,f=a+1;continue}if(l===13){e.charCodeAt(a+1)!==10&&(c++,f=a+1);continue}let E=a,F=l,p;if(l>=55296&&l<=56319){let t=e.charCodeAt(a+1);if(!(t>=56320&&t<=57343))continue;F=65536+(l-55296<<10)+(t-56320),p=I(F),a++}else p=A(l);if(p!==null&&(o[p]++,s++,i.length<u)){let t={offset:E,line:c,column:E-f+1,codePoint:F,category:p};p==="bom"&&(t.atStart=E===0),i.push(t)}}return{findings:i,counts:o,total:s,truncated:s>i.length}}function b(e){if(typeof e!="string")throw new TypeError("getNewlineStats expects a string input.");let r=0,n=0,u=0;for(let s=0,c=e.length;s<c;s++){let f=e.charCodeAt(s);f===13?e.charCodeAt(s+1)===10?(r++,s++):n++:f===10&&u++}let o={crlf:r,cr:n,lf:u},i=[];return o.crlf>0&&i.push("CRLF"),o.cr>0&&i.push("CR"),o.lf>0&&i.push("LF"),{...o,types:i,mixed:i.length>1}}function K(e){return{purgeInvisibleChars:e.hasInvisibleChars,purgeEmojis:e.hasEmojis,nukeControls:e.hasControlChars,keyboardOnlyFilter:e.hasNonKeyboardChars,normalizeNewlines:e.hasMixedNewlines||e.newlineStyle==="CRLF"||e.newlineStyle==="CR"}}function H(e={}){let r=!!e.normalizeSpaces,n=!!e.purgeZeroWidthSpaces,u=!!e.purgeInvisibleChars,o=!!e.purgeEmojis,i=!!e.nukeControls,s=!!e.keyboardOnlyFilter,c=!!e.normalizeNewlines,f=!!e.trimSpacesAroundNewlines,a=!!e.collapseNewLines,h=!!e.preserveParagraphs,l=!!e.collapseSpaces,E=!!e.finalTrim,F=Number.isFinite(e.maxLength)&&e.maxLength>=0?e.maxLength:1/0,p=e.onMaxLength==="truncate"||e.onMaxLength==="noop"||e.onMaxLength==="throw"?e.onMaxLength:"throw";return t=>_(t,u,o,i,s,c,f,a,h,l,E,F,p,r,n)}function R(e){return _(e,!0,!0,!0,!1,!0,!0,!0,!1,!0,!0)}function w(e){return _(e,!1,!1,!1,!1,!0,!0,!0,!0,!0,!0)}function N(e){return _(e,!1,!1,!1,!0,!0,!0,!1,!1,!1,!0)}function T(e){return _(e,!0,!0,!0,!0,!0,!0,!0,!1,!0,!0)}function L(e){return _(e,!1,!1,!1,!1,!0,!1,!1,!1,!1,!1,1/0,"throw",!0,!0)}var W=Object.assign(H,{strict:R,loose:w,keyboardOnlyEmoji:N,keyboardOnly:T,pasted:L});function _(e,r=!1,n=!1,u=!1,o=!1,i=!1,s=!1,c=!1,f=!1,a=!1,h=!1,l=1/0,E="throw",F=!1,p=!1){if(typeof e!="string")throw new TypeError("sanctifyText expects a string input.");if(E!=="truncate"&&E!=="noop"&&E!=="throw"&&(E="throw"),e.length>l)switch(E){case"truncate":e=e.slice(0,l);break;case"noop":return e;case"throw":default:throw new RangeError(`sanctifyText input length ${e.length} exceeds maxLength ${l}.`)}let t=e;return F&&(t=t.replace(J," ")),p&&(t=t.replace(V,"")),r&&(t=Z(t)),n&&(t=q(t)),u&&(t=le(t)),o&&(t=Y(t,n)),i&&(t=ee(t)),s&&(t=ne(t)),c&&(t=oe(t,f)),a&&(t=se(t)),h?t.trim():t}var J=/[\u00A0\u2000-\u200A\u202F\u205F]/g,V=/[\u200B\u2060\uFEFF]/g;function Z(e){return e.replace(d,"")}function $(e){if(x.lastIndex=0,!x.test(e))return"";x.lastIndex=0;let r="";for(let n of e.matchAll(x))r+=n[0];return r}function Y(e,r=!1){let n=S(e);return r?n.replace(g,""):n.replace(g,u=>$(u))}function q(e){return e.replace(x,"")}var Q=/\r\n|\r|\n/g;function ee(e,r=`
|
|
3
|
+
`){return e.replace(Q,r)}var re=/[ \t]*(\n+)[ \t]*/g;function ne(e){return e.replace(re,"$1")}var te=/\n{2,}/g,ie=/\n{3,}/g;function oe(e,r){return r?e.replace(ie,`
|
|
3
4
|
|
|
4
|
-
`):e.replace(
|
|
5
|
-
`)}var
|
|
6
|
-
`||n==="\r"||n>=" "&&n<="~")&&!m(p,n))return!0;return!1}function Y(e){if(typeof e!="string")throw new TypeError("inspectText expects a string input.");let r=[],n={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:r},o=(l,c,f)=>{l&&(n[c]=!0,r.push(f))};o(m(d,e),"hasControlChars","Control characters detected."),o(m(C,e),"hasInvisibleChars","Invisible Unicode characters detected."),o(m(p,e),"hasEmojis","Emojis detected.");let{mixed:u,types:i}=w(e);return n.hasMixedNewlines=u,n.newlineStyle=u?"Mixed":i[0]||null,n.newlineStyle&&r.push(u?"Mixed newline styles detected.":`Consistent newline style: ${n.newlineStyle}`),o(Q(e),"hasNonKeyboardChars","Non-keyboard characters detected."),n}var ee=100;function y(e,r={}){if(typeof e!="string")throw new TypeError("inspectTextDetailed expects a string input.");let n=r&&r.maxFindings,o=n===1/0||Number.isInteger(n)&&n>=0?n:ee,u={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},i=[],l=0,c=1,f=0;for(let a=0,g=e.length;a<g;a++){let s=e.charCodeAt(a);if(s>=32&&s<127)continue;if(s===10){c++,f=a+1;continue}if(s===13){e.charCodeAt(a+1)!==10&&(c++,f=a+1);continue}let E=a,F=s,x;if(s>=55296&&s<=56319){let t=e.charCodeAt(a+1);if(!(t>=56320&&t<=57343))continue;F=65536+(s-55296<<10)+(t-56320),x=b(F),a++}else x=I(s);if(x!==null&&(u[x]++,l++,i.length<o)){let t={offset:E,line:c,column:E-f+1,codePoint:F,category:x};x==="bom"&&(t.atStart=E===0),i.push(t)}}return{findings:i,counts:u,total:l,truncated:l>i.length}}function w(e){if(typeof e!="string")throw new TypeError("getNewlineStats expects a string input.");let r=0,n=0,o=0;for(let l=0,c=e.length;l<c;l++){let f=e.charCodeAt(l);f===13?e.charCodeAt(l+1)===10?(r++,l++):n++:f===10&&o++}let u={crlf:r,cr:n,lf:o},i=[];return u.crlf>0&&i.push("CRLF"),u.cr>0&&i.push("CR"),u.lf>0&&i.push("LF"),{...u,types:i,mixed:i.length>1}}function re(e){return{purgeInvisibleChars:e.hasInvisibleChars,purgeEmojis:e.hasEmojis,nukeControls:e.hasControlChars,keyboardOnlyFilter:e.hasNonKeyboardChars,normalizeNewlines:e.hasMixedNewlines||e.newlineStyle==="CRLF"||e.newlineStyle==="CR"}}function ne(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",o="",u=0;for(let i of y(e,{maxFindings:1/0}).findings)o+=e.slice(u,i.offset)+n(i.codePoint,i.category),u=i.offset+(i.codePoint>65535?2:1);return o+e.slice(u)}export{w as getNewlineStats,re as getRecommendedSanctifierOptions,Y as inspectText,y as inspectTextDetailed,ne as revealInvisible,h as summonSanctifier};
|
|
5
|
+
`):e.replace(te,`
|
|
6
|
+
`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.6";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};
|
|
@@ -7,17 +7,26 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export function decodeTagText(text: string): string;
|
|
9
9
|
/**
|
|
10
|
-
* Finds runs of tag characters, variation selectors
|
|
10
|
+
* Finds runs of tag characters, variation selectors, invisible operators and reserved
|
|
11
|
+
* tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
|
|
11
12
|
* Consecutive characters of the same kind form one run. Two kinds of sequence are
|
|
12
13
|
* legitimate and listed under `benign` instead of `runs`:
|
|
13
14
|
* - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
|
|
14
15
|
* tags for "gbeng", "gbsct" or "gbwls" and U+E007F
|
|
15
16
|
* - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
|
|
17
|
+
*
|
|
18
|
+
* `line` and `column` are counted as in inspectTextDetailed. Every run and benign
|
|
19
|
+
* span is counted; only the first `maxRuns` of each are listed. With the caps set,
|
|
20
|
+
* memory stays flat however large the input is. One pass over the string, no copy.
|
|
21
|
+
* Set both caps for untrusted input: without them, results grow with the input.
|
|
22
|
+
*
|
|
16
23
|
* @param {string} text
|
|
24
|
+
* @param {HiddenScanOptions} [options]
|
|
17
25
|
* @returns {HiddenTextScan}
|
|
18
26
|
*/
|
|
19
|
-
export function findHiddenRuns(text: string): HiddenTextScan;
|
|
20
|
-
export
|
|
27
|
+
export function findHiddenRuns(text: string, options?: HiddenScanOptions): HiddenTextScan;
|
|
28
|
+
export { VERSION } from "./version.generated.js";
|
|
29
|
+
export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator" | "reserved_tag_plane";
|
|
21
30
|
export type HiddenRun = {
|
|
22
31
|
kind: HiddenRunKind;
|
|
23
32
|
/**
|
|
@@ -33,18 +42,65 @@ export type HiddenRun = {
|
|
|
33
42
|
*/
|
|
34
43
|
count: number;
|
|
35
44
|
/**
|
|
36
|
-
* -
|
|
45
|
+
* - 1-based line of the first character (as in inspectTextDetailed)
|
|
46
|
+
*/
|
|
47
|
+
line: number;
|
|
48
|
+
/**
|
|
49
|
+
* - 1-based column of the first character, in UTF-16 units
|
|
50
|
+
*/
|
|
51
|
+
column: number;
|
|
52
|
+
/**
|
|
53
|
+
* - tag runs only: the ASCII the tags spell (see decodeTagText),
|
|
54
|
+
* at most `maxDecodedLength` UTF-16 units
|
|
37
55
|
*/
|
|
38
56
|
decoded?: string;
|
|
57
|
+
/**
|
|
58
|
+
* - tag runs only: true when `decoded` stops short of the run
|
|
59
|
+
*/
|
|
60
|
+
decodedTruncated?: boolean;
|
|
61
|
+
};
|
|
62
|
+
export type HiddenRunCounts = {
|
|
63
|
+
tag: number;
|
|
64
|
+
variation_selector: number;
|
|
65
|
+
invisible_operator: number;
|
|
66
|
+
reserved_tag_plane: number;
|
|
39
67
|
};
|
|
40
68
|
export type HiddenTextScan = {
|
|
41
69
|
/**
|
|
42
|
-
* - runs that may hide text, in order
|
|
70
|
+
* - runs that may hide text, in order, at most `maxRuns` long
|
|
43
71
|
*/
|
|
44
72
|
runs: HiddenRun[];
|
|
45
73
|
/**
|
|
46
74
|
* - [offset, end) spans of legitimate
|
|
47
|
-
* sequences (subdivision flags, a single emoji or text presentation selector)
|
|
75
|
+
* sequences (subdivision flags, a single emoji or text presentation selector), at most `maxRuns` long
|
|
48
76
|
*/
|
|
49
77
|
benign: Array<[number, number]>;
|
|
78
|
+
/**
|
|
79
|
+
* - every run per kind, listed or not
|
|
80
|
+
*/
|
|
81
|
+
counts: HiddenRunCounts;
|
|
82
|
+
/**
|
|
83
|
+
* - sum of `counts`
|
|
84
|
+
*/
|
|
85
|
+
total: number;
|
|
86
|
+
/**
|
|
87
|
+
* - every benign span, listed or not
|
|
88
|
+
*/
|
|
89
|
+
benignTotal: number;
|
|
90
|
+
/**
|
|
91
|
+
* - true when runs or benign spans were left out of the lists
|
|
92
|
+
*/
|
|
93
|
+
truncated: boolean;
|
|
94
|
+
};
|
|
95
|
+
export type HiddenScanOptions = {
|
|
96
|
+
/**
|
|
97
|
+
* - cap on listed runs, and separately on listed benign spans
|
|
98
|
+
* (non-negative integer or Infinity). Counts stay exact.
|
|
99
|
+
*/
|
|
100
|
+
maxRuns?: number;
|
|
101
|
+
/**
|
|
102
|
+
* - cap on each run's `decoded` text, in UTF-16 units.
|
|
103
|
+
* For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
|
|
104
|
+
*/
|
|
105
|
+
maxDecodedLength?: number;
|
|
50
106
|
};
|
package/dist/types/index.d.ts
CHANGED
|
@@ -12,5 +12,11 @@ import { inspectTextDetailed } from './inspectText.js';
|
|
|
12
12
|
import { getNewlineStats } from './inspectText.js';
|
|
13
13
|
import { getRecommendedSanctifierOptions } from './inspectText.js';
|
|
14
14
|
import { summonSanctifier } from './sanctifyText.js';
|
|
15
|
+
import { sanctifyStrict } from './sanctifyText.js';
|
|
16
|
+
import { sanctifyLoose } from './sanctifyText.js';
|
|
17
|
+
import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
|
|
18
|
+
import { sanctifyKeyboardOnly } from './sanctifyText.js';
|
|
19
|
+
import { sanctifyPasted } from './sanctifyText.js';
|
|
15
20
|
import { revealInvisible } from './reveal.js';
|
|
16
|
-
|
|
21
|
+
import { VERSION } from './version.generated.js';
|
|
22
|
+
export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
|
|
@@ -1,103 +1,51 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* @
|
|
8
|
-
* @
|
|
9
|
-
* @property {boolean} [collapseNewLines]
|
|
10
|
-
* @property {boolean} [preserveParagraphs]
|
|
11
|
-
* @property {boolean} [collapseSpaces]
|
|
12
|
-
* @property {boolean} [finalTrim]
|
|
13
|
-
* @property {number} [maxLength]
|
|
14
|
-
* @property {'throw'|'truncate'|'noop'} [onMaxLength]
|
|
15
|
-
* @property {boolean} [normalizeSpaces] - Replace look-alike spaces with an ordinary space (runs first).
|
|
16
|
-
* @property {boolean} [purgeZeroWidthSpaces] - Remove U+200B, U+2060 and U+FEFF only, keeping joiners and selectors.
|
|
2
|
+
* Strict sanitizer:
|
|
3
|
+
* - Purge emojis
|
|
4
|
+
* - Collapse all newlines
|
|
5
|
+
* - Collapse spaces
|
|
6
|
+
* - Nuke control characters
|
|
7
|
+
* @param {string} text
|
|
8
|
+
* @returns {string}
|
|
17
9
|
*/
|
|
10
|
+
export function sanctifyStrict(text: string): string;
|
|
18
11
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* @param {
|
|
24
|
-
* @
|
|
25
|
-
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
26
|
-
* @param {boolean} [defaultOptions.nukeControls]
|
|
27
|
-
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
28
|
-
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
29
|
-
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
30
|
-
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
31
|
-
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
32
|
-
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
33
|
-
* @param {boolean} [defaultOptions.finalTrim]
|
|
34
|
-
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
35
|
-
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
36
|
-
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
37
|
-
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
38
|
-
* @returns {(text: string) => string}
|
|
12
|
+
* Loose sanitizer:
|
|
13
|
+
* - Collapse spaces
|
|
14
|
+
* - Preserve paragraphs
|
|
15
|
+
* - Normalize newlines
|
|
16
|
+
* @param {string} text
|
|
17
|
+
* @returns {string}
|
|
39
18
|
*/
|
|
40
|
-
export function
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
* - Normalize newlines
|
|
71
|
-
*/
|
|
72
|
-
/** @param {string} text @returns {string} */
|
|
73
|
-
function loose(text: string): string;
|
|
74
|
-
/**
|
|
75
|
-
* Keyboard-only (with emojis):
|
|
76
|
-
* - Keeps emojis and printable ASCII
|
|
77
|
-
* - Strips non-standard characters
|
|
78
|
-
* - Normalizes typographic trash
|
|
79
|
-
*/
|
|
80
|
-
/** @param {string} text @returns {string} */
|
|
81
|
-
function keyboardOnlyEmoji(text: string): string;
|
|
82
|
-
/**
|
|
83
|
-
* Keyboard-only (strict):
|
|
84
|
-
* - Removes emojis
|
|
85
|
-
* - Collapses all whitespace
|
|
86
|
-
* - Restricts to printable ASCII only
|
|
87
|
-
*/
|
|
88
|
-
/** @param {string} text @returns {string} */
|
|
89
|
-
function keyboardOnly(text: string): string;
|
|
90
|
-
/**
|
|
91
|
-
* Pasted text and model output:
|
|
92
|
-
* - Look-alike spaces (NBSP, narrow NBSP, typographic spaces) become ordinary spaces,
|
|
93
|
-
* so words never run together
|
|
94
|
-
* - Zero-width spaces, word joiners and BOMs are removed
|
|
95
|
-
* - Newlines are normalized to `\n`
|
|
96
|
-
* Keeps emoji joiners, variation selectors, bidi marks, indentation and line breaks.
|
|
97
|
-
*/
|
|
98
|
-
/** @param {string} text @returns {string} */
|
|
99
|
-
function pasted(text: string): string;
|
|
100
|
-
}
|
|
19
|
+
export function sanctifyLoose(text: string): string;
|
|
20
|
+
/**
|
|
21
|
+
* Keyboard-only (with emojis):
|
|
22
|
+
* - Keeps emojis and printable ASCII
|
|
23
|
+
* - Strips non-standard characters
|
|
24
|
+
* - Normalizes typographic trash
|
|
25
|
+
* @param {string} text
|
|
26
|
+
* @returns {string}
|
|
27
|
+
*/
|
|
28
|
+
export function sanctifyKeyboardOnlyEmoji(text: string): string;
|
|
29
|
+
/**
|
|
30
|
+
* Keyboard-only (strict):
|
|
31
|
+
* - Removes emojis
|
|
32
|
+
* - Collapses all whitespace
|
|
33
|
+
* - Restricts to printable ASCII only
|
|
34
|
+
* @param {string} text
|
|
35
|
+
* @returns {string}
|
|
36
|
+
*/
|
|
37
|
+
export function sanctifyKeyboardOnly(text: string): string;
|
|
38
|
+
/**
|
|
39
|
+
* Pasted text and model output:
|
|
40
|
+
* - Look-alike spaces (NBSP, narrow NBSP, typographic spaces) become ordinary spaces,
|
|
41
|
+
* so words never run together
|
|
42
|
+
* - Zero-width spaces, word joiners and BOMs are removed
|
|
43
|
+
* - Newlines are normalized to `\n`
|
|
44
|
+
* Keeps emoji joiners, variation selectors, bidi marks, indentation and line breaks.
|
|
45
|
+
* @param {string} text
|
|
46
|
+
* @returns {string}
|
|
47
|
+
*/
|
|
48
|
+
export function sanctifyPasted(text: string): string;
|
|
101
49
|
/**
|
|
102
50
|
* Text Sanctifier
|
|
103
51
|
*
|
|
@@ -131,11 +79,45 @@ export namespace summonSanctifier {
|
|
|
131
79
|
*/
|
|
132
80
|
export function sanctifyText(text: string, purgeInvisibleChars?: boolean, purgeEmojis?: boolean, nukeControls?: boolean, keyboardOnlyFilter?: boolean, normalizeNewlines?: boolean, trimSpacesAroundNewlines?: boolean, collapseNewLines?: boolean, preserveParagraphs?: boolean, collapseSpaces?: boolean, finalTrim?: boolean, maxLength?: number, onMaxLength?: "throw" | "truncate" | "noop", normalizeSpaces?: boolean, purgeZeroWidthSpaces?: boolean): string;
|
|
133
81
|
/**
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
82
|
+
* Summons a customized sanctifier function with pre-bound booleans.
|
|
83
|
+
*
|
|
84
|
+
* Accepts full flag names and returns a text-cleaning function.
|
|
85
|
+
*
|
|
86
|
+
* @param {Object} [defaultOptions={}]
|
|
87
|
+
* @param {boolean} [defaultOptions.purgeInvisibleChars]
|
|
88
|
+
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
89
|
+
* @param {boolean} [defaultOptions.nukeControls]
|
|
90
|
+
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
91
|
+
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
92
|
+
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
93
|
+
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
94
|
+
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
95
|
+
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
96
|
+
* @param {boolean} [defaultOptions.finalTrim]
|
|
97
|
+
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
98
|
+
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
99
|
+
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
100
|
+
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
101
|
+
* @returns {(text: string) => string}
|
|
102
|
+
*
|
|
103
|
+
* The presets are attached as properties (`summonSanctifier.strict`, `.loose`,
|
|
104
|
+
* `.keyboardOnlyEmoji`, `.keyboardOnly`, `.pasted`) and are also standalone exports.
|
|
105
|
+
* Built in a call marked pure, so a bundle that imports only a preset leaves it out.
|
|
106
|
+
* @type {typeof createSanctifier & {
|
|
107
|
+
* strict: typeof sanctifyStrict,
|
|
108
|
+
* loose: typeof sanctifyLoose,
|
|
109
|
+
* keyboardOnlyEmoji: typeof sanctifyKeyboardOnlyEmoji,
|
|
110
|
+
* keyboardOnly: typeof sanctifyKeyboardOnly,
|
|
111
|
+
* pasted: typeof sanctifyPasted,
|
|
112
|
+
* }}
|
|
137
113
|
*/
|
|
138
|
-
export
|
|
114
|
+
export const summonSanctifier: typeof createSanctifier & {
|
|
115
|
+
strict: typeof sanctifyStrict;
|
|
116
|
+
loose: typeof sanctifyLoose;
|
|
117
|
+
keyboardOnlyEmoji: typeof sanctifyKeyboardOnlyEmoji;
|
|
118
|
+
keyboardOnly: typeof sanctifyKeyboardOnly;
|
|
119
|
+
pasted: typeof sanctifyPasted;
|
|
120
|
+
};
|
|
139
121
|
/**
|
|
140
122
|
* Look-alike spaces: the `nonbreaking_space` category except U+3000, the ordinary
|
|
141
123
|
* space of CJK text. A test checks this against the category table.
|
|
@@ -143,15 +125,6 @@ export function normalizeTypographicJank(text: string): string;
|
|
|
143
125
|
export const SPACE_LOOKALIKES_REGEX: RegExp;
|
|
144
126
|
/** Zero-width space, word joiner and BOM: invisible and never needed in plain text. */
|
|
145
127
|
export const ZERO_WIDTH_SPACES_REGEX: RegExp;
|
|
146
|
-
/**
|
|
147
|
-
* Matches any character that is NOT:
|
|
148
|
-
* - Printable ASCII (U+0020–U+007E)
|
|
149
|
-
* - Newline characters: \n (LF), \r (CR)
|
|
150
|
-
* This allows for \n, \r, and \r\n line endings.
|
|
151
|
-
*/
|
|
152
|
-
export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
|
|
153
|
-
/** @type {RegExp} */
|
|
154
|
-
export let EMOJI_REGEX: RegExp;
|
|
155
128
|
export type SanctifyOptions = {
|
|
156
129
|
purgeInvisibleChars?: boolean;
|
|
157
130
|
purgeEmojis?: boolean;
|
|
@@ -176,4 +149,62 @@ export type SanctifyOptions = {
|
|
|
176
149
|
};
|
|
177
150
|
import { CONTROL_CHARS_REGEX } from './categories.generated.js';
|
|
178
151
|
import { INVISIBLE_TRASH_REGEX } from './categories.generated.js';
|
|
179
|
-
|
|
152
|
+
import { ASCII_KEYBOARD_SAFE_REGEX } from './textPatterns.js';
|
|
153
|
+
import { EMOJI_REGEX } from './textPatterns.js';
|
|
154
|
+
import { normalizeTypographicJank } from './textPatterns.js';
|
|
155
|
+
/**
|
|
156
|
+
* @typedef {Object} SanctifyOptions
|
|
157
|
+
* @property {boolean} [purgeInvisibleChars]
|
|
158
|
+
* @property {boolean} [purgeEmojis]
|
|
159
|
+
* @property {boolean} [nukeControls]
|
|
160
|
+
* @property {boolean} [keyboardOnlyFilter]
|
|
161
|
+
* @property {boolean} [normalizeNewlines]
|
|
162
|
+
* @property {boolean} [trimSpacesAroundNewlines]
|
|
163
|
+
* @property {boolean} [collapseNewLines]
|
|
164
|
+
* @property {boolean} [preserveParagraphs]
|
|
165
|
+
* @property {boolean} [collapseSpaces]
|
|
166
|
+
* @property {boolean} [finalTrim]
|
|
167
|
+
* @property {number} [maxLength]
|
|
168
|
+
* @property {'throw'|'truncate'|'noop'} [onMaxLength]
|
|
169
|
+
* @property {boolean} [normalizeSpaces] - Replace look-alike spaces with an ordinary space (runs first).
|
|
170
|
+
* @property {boolean} [purgeZeroWidthSpaces] - Remove U+200B, U+2060 and U+FEFF only, keeping joiners and selectors.
|
|
171
|
+
*/
|
|
172
|
+
/**
|
|
173
|
+
* Summons a customized sanctifier function with pre-bound booleans.
|
|
174
|
+
*
|
|
175
|
+
* Accepts full flag names and returns a text-cleaning function.
|
|
176
|
+
*
|
|
177
|
+
* @param {Object} [defaultOptions={}]
|
|
178
|
+
* @param {boolean} [defaultOptions.purgeInvisibleChars]
|
|
179
|
+
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
180
|
+
* @param {boolean} [defaultOptions.nukeControls]
|
|
181
|
+
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
182
|
+
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
183
|
+
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
184
|
+
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
185
|
+
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
186
|
+
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
187
|
+
* @param {boolean} [defaultOptions.finalTrim]
|
|
188
|
+
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
189
|
+
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
190
|
+
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
191
|
+
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
192
|
+
* @returns {(text: string) => string}
|
|
193
|
+
*/
|
|
194
|
+
declare function createSanctifier(defaultOptions?: {
|
|
195
|
+
purgeInvisibleChars?: boolean;
|
|
196
|
+
purgeEmojis?: boolean;
|
|
197
|
+
nukeControls?: boolean;
|
|
198
|
+
keyboardOnlyFilter?: boolean;
|
|
199
|
+
normalizeNewlines?: boolean;
|
|
200
|
+
trimSpacesAroundNewlines?: boolean;
|
|
201
|
+
collapseNewLines?: boolean;
|
|
202
|
+
preserveParagraphs?: boolean;
|
|
203
|
+
collapseSpaces?: boolean;
|
|
204
|
+
finalTrim?: boolean;
|
|
205
|
+
maxLength?: number;
|
|
206
|
+
onMaxLength?: "throw" | "truncate" | "noop";
|
|
207
|
+
normalizeSpaces?: boolean;
|
|
208
|
+
purgeZeroWidthSpaces?: boolean;
|
|
209
|
+
}): (text: string) => string;
|
|
210
|
+
export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX, ASCII_KEYBOARD_SAFE_REGEX, EMOJI_REGEX, normalizeTypographicJank };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Normalizes typographic Unicode punctuation into ASCII equivalents.
|
|
3
|
+
* @param {string} text
|
|
4
|
+
* @returns {string}
|
|
5
|
+
*/
|
|
6
|
+
export function normalizeTypographicJank(text: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* Matches any character that is NOT:
|
|
9
|
+
* - Printable ASCII (U+0020–U+007E)
|
|
10
|
+
* - Newline characters: \n (LF), \r (CR)
|
|
11
|
+
* This allows for \n, \r, and \r\n line endings.
|
|
12
|
+
*/
|
|
13
|
+
export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
|
|
14
|
+
/** @type {RegExp} */
|
|
15
|
+
export const EMOJI_REGEX: RegExp;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "text-sanctifier",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
|
|
6
6
|
"engines": {
|
|
@@ -39,12 +39,12 @@
|
|
|
39
39
|
"types": "./dist/types/index.d.ts",
|
|
40
40
|
"sideEffects": false,
|
|
41
41
|
"scripts": {
|
|
42
|
-
"gen": "node scripts/gen-categories.js",
|
|
43
|
-
"gen:check": "node scripts/gen-categories.js --check",
|
|
42
|
+
"gen": "node scripts/gen-categories.js && node scripts/gen-version.js",
|
|
43
|
+
"gen:check": "node scripts/gen-categories.js --check && node scripts/gen-version.js --check",
|
|
44
44
|
"build": "node scripts/build.js",
|
|
45
45
|
"build:types": "tsc -p tsconfig.types.json",
|
|
46
46
|
"test:types": "tsc -p tests/types/tsconfig.json",
|
|
47
|
-
"test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node tests/cli.test.js",
|
|
47
|
+
"test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node tests/cli.test.js",
|
|
48
48
|
"test-min": "node tests/sanctifyText.test.min.js",
|
|
49
49
|
"ci": "npm run gen:check && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
|
|
50
50
|
"size": "node scripts/check-size.js"
|
package/src/hiddenText.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
// src/hiddenText.js
|
|
2
2
|
//
|
|
3
3
|
// Finds runs of the characters used to hide instructions in text a model reads:
|
|
4
|
-
// tag characters, variation selectors
|
|
4
|
+
// tag characters, variation selectors, invisible operators and the reserved code points
|
|
5
|
+
// of the tag plane. Tag runs are decoded
|
|
5
6
|
// to the ASCII they spell. Used by `text-sanctifier check --ai-config`, and exported
|
|
6
7
|
// as `text-sanctifier/hidden`.
|
|
7
8
|
//
|
|
8
9
|
// Plain JavaScript with no Node APIs, so it also runs in a browser. src/index.js does
|
|
9
10
|
// not import this file, so it stays out of the browser bundle and its size budget.
|
|
10
11
|
|
|
11
|
-
|
|
12
|
+
export { VERSION } from './version.generated.js';
|
|
13
|
+
|
|
14
|
+
/** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
|
|
12
15
|
|
|
13
16
|
/**
|
|
14
17
|
* @typedef {Object} HiddenRun
|
|
@@ -16,14 +19,38 @@
|
|
|
16
19
|
* @property {number} offset - UTF-16 index of the first character
|
|
17
20
|
* @property {number} end - UTF-16 index just past the last character
|
|
18
21
|
* @property {number} count - number of code points in the run
|
|
19
|
-
* @property {
|
|
22
|
+
* @property {number} line - 1-based line of the first character (as in inspectTextDetailed)
|
|
23
|
+
* @property {number} column - 1-based column of the first character, in UTF-16 units
|
|
24
|
+
* @property {string} [decoded] - tag runs only: the ASCII the tags spell (see decodeTagText),
|
|
25
|
+
* at most `maxDecodedLength` UTF-16 units
|
|
26
|
+
* @property {boolean} [decodedTruncated] - tag runs only: true when `decoded` stops short of the run
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @typedef {Object} HiddenRunCounts
|
|
31
|
+
* @property {number} tag
|
|
32
|
+
* @property {number} variation_selector
|
|
33
|
+
* @property {number} invisible_operator
|
|
34
|
+
* @property {number} reserved_tag_plane
|
|
20
35
|
*/
|
|
21
36
|
|
|
22
37
|
/**
|
|
23
38
|
* @typedef {Object} HiddenTextScan
|
|
24
|
-
* @property {HiddenRun[]} runs - runs that may hide text, in order
|
|
39
|
+
* @property {HiddenRun[]} runs - runs that may hide text, in order, at most `maxRuns` long
|
|
25
40
|
* @property {Array<[number, number]>} benign - [offset, end) spans of legitimate
|
|
26
|
-
* sequences (subdivision flags, a single emoji or text presentation selector)
|
|
41
|
+
* sequences (subdivision flags, a single emoji or text presentation selector), at most `maxRuns` long
|
|
42
|
+
* @property {HiddenRunCounts} counts - every run per kind, listed or not
|
|
43
|
+
* @property {number} total - sum of `counts`
|
|
44
|
+
* @property {number} benignTotal - every benign span, listed or not
|
|
45
|
+
* @property {boolean} truncated - true when runs or benign spans were left out of the lists
|
|
46
|
+
*/
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* @typedef {Object} HiddenScanOptions
|
|
50
|
+
* @property {number} [maxRuns=Infinity] - cap on listed runs, and separately on listed benign spans
|
|
51
|
+
* (non-negative integer or Infinity). Counts stay exact.
|
|
52
|
+
* @property {number} [maxDecodedLength=Infinity] - cap on each run's `decoded` text, in UTF-16 units.
|
|
53
|
+
* For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
|
|
27
54
|
*/
|
|
28
55
|
|
|
29
56
|
const TAG_FIRST = 0xE0000;
|
|
@@ -43,6 +70,8 @@ function kindOf(cp) {
|
|
|
43
70
|
if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
|
|
44
71
|
(cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
|
|
45
72
|
if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
|
|
73
|
+
// Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
|
|
74
|
+
if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
|
|
46
75
|
return null;
|
|
47
76
|
}
|
|
48
77
|
|
|
@@ -57,35 +86,101 @@ const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
|
|
|
57
86
|
* @returns {string}
|
|
58
87
|
*/
|
|
59
88
|
export function decodeTagText(text) {
|
|
89
|
+
return decodeRange(text, 0, text.length, Infinity).decoded;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Decodes text[start, end) as decodeTagText does, stopping before the output would
|
|
94
|
+
* pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
|
|
95
|
+
* @param {string} text
|
|
96
|
+
* @param {number} start
|
|
97
|
+
* @param {number} end
|
|
98
|
+
* @param {number} limit
|
|
99
|
+
* @returns {{ decoded: string, truncated: boolean }}
|
|
100
|
+
*/
|
|
101
|
+
function decodeRange(text, start, end, limit) {
|
|
60
102
|
let out = '';
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
103
|
+
let i = start;
|
|
104
|
+
while (i < end) {
|
|
105
|
+
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
106
|
+
const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
|
|
107
|
+
if (out.length + piece.length > limit) return { decoded: out, truncated: true };
|
|
108
|
+
out += piece;
|
|
109
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
64
110
|
}
|
|
65
|
-
return out;
|
|
111
|
+
return { decoded: out, truncated: false };
|
|
66
112
|
}
|
|
67
113
|
|
|
114
|
+
// Whitespace as JavaScript's \s defines it, tested at an index of the text itself,
|
|
115
|
+
// so the check allocates nothing.
|
|
116
|
+
const SPACE_AT = /\s/y;
|
|
117
|
+
|
|
68
118
|
/**
|
|
69
|
-
*
|
|
119
|
+
* @param {string} text
|
|
120
|
+
* @param {number} index
|
|
121
|
+
* @returns {boolean}
|
|
122
|
+
*/
|
|
123
|
+
function isSpaceAt(text, index) {
|
|
124
|
+
SPACE_AT.lastIndex = index;
|
|
125
|
+
return SPACE_AT.test(text);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* A cap option: a non-negative integer or Infinity; anything else means no cap.
|
|
130
|
+
* @param {unknown} value
|
|
131
|
+
* @returns {number}
|
|
132
|
+
*/
|
|
133
|
+
function cap(value) {
|
|
134
|
+
return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
|
|
135
|
+
? value
|
|
136
|
+
: Infinity;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Finds runs of tag characters, variation selectors, invisible operators and reserved
|
|
141
|
+
* tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
|
|
70
142
|
* Consecutive characters of the same kind form one run. Two kinds of sequence are
|
|
71
143
|
* legitimate and listed under `benign` instead of `runs`:
|
|
72
144
|
* - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
|
|
73
145
|
* tags for "gbeng", "gbsct" or "gbwls" and U+E007F
|
|
74
146
|
* - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
|
|
147
|
+
*
|
|
148
|
+
* `line` and `column` are counted as in inspectTextDetailed. Every run and benign
|
|
149
|
+
* span is counted; only the first `maxRuns` of each are listed. With the caps set,
|
|
150
|
+
* memory stays flat however large the input is. One pass over the string, no copy.
|
|
151
|
+
* Set both caps for untrusted input: without them, results grow with the input.
|
|
152
|
+
*
|
|
75
153
|
* @param {string} text
|
|
154
|
+
* @param {HiddenScanOptions} [options]
|
|
76
155
|
* @returns {HiddenTextScan}
|
|
77
156
|
*/
|
|
78
|
-
export function findHiddenRuns(text) {
|
|
157
|
+
export function findHiddenRuns(text, options = {}) {
|
|
158
|
+
const maxRuns = cap(options && options.maxRuns);
|
|
159
|
+
const maxDecodedLength = cap(options && options.maxDecodedLength);
|
|
160
|
+
|
|
79
161
|
/** @type {HiddenRun[]} */
|
|
80
162
|
const runs = [];
|
|
81
163
|
/** @type {Array<[number, number]>} */
|
|
82
164
|
const benign = [];
|
|
165
|
+
/** @type {HiddenRunCounts} */
|
|
166
|
+
const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
|
|
167
|
+
let total = 0;
|
|
168
|
+
let benignTotal = 0;
|
|
169
|
+
let line = 1;
|
|
170
|
+
let lineStart = 0;
|
|
83
171
|
|
|
84
172
|
let i = 0;
|
|
85
173
|
while (i < text.length) {
|
|
86
174
|
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
87
175
|
const kind = kindOf(cp);
|
|
88
|
-
if (kind === null) {
|
|
176
|
+
if (kind === null) {
|
|
177
|
+
if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
|
|
178
|
+
line++;
|
|
179
|
+
lineStart = i + 1;
|
|
180
|
+
}
|
|
181
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
89
184
|
|
|
90
185
|
const offset = i;
|
|
91
186
|
let count = 0;
|
|
@@ -99,22 +194,30 @@ export function findHiddenRuns(text) {
|
|
|
99
194
|
}
|
|
100
195
|
const end = i;
|
|
101
196
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
197
|
+
// A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
|
|
198
|
+
const isBenign = kind === 'tag'
|
|
199
|
+
? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
|
|
200
|
+
SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
|
|
201
|
+
: kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
|
|
202
|
+
offset > 0 && !isSpaceAt(text, offset - 1);
|
|
203
|
+
if (isBenign) {
|
|
204
|
+
benignTotal++;
|
|
205
|
+
if (benign.length < maxRuns) benign.push([offset, end]);
|
|
110
206
|
continue;
|
|
111
207
|
}
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
208
|
+
|
|
209
|
+
counts[kind]++;
|
|
210
|
+
total++;
|
|
211
|
+
if (runs.length < maxRuns) {
|
|
212
|
+
/** @type {HiddenRun} */
|
|
213
|
+
const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
|
|
214
|
+
if (kind === 'tag') {
|
|
215
|
+
const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
|
|
216
|
+
run.decoded = decoded;
|
|
217
|
+
run.decodedTruncated = truncated;
|
|
218
|
+
}
|
|
219
|
+
runs.push(run);
|
|
116
220
|
}
|
|
117
|
-
runs.push({ kind, offset, end, count });
|
|
118
221
|
}
|
|
119
|
-
return { runs, benign };
|
|
222
|
+
return { runs, benign, counts, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
|
|
120
223
|
}
|
package/src/index.d.ts
CHANGED
|
@@ -3,8 +3,14 @@ import { inspectTextDetailed } from './inspectText.js';
|
|
|
3
3
|
import { getNewlineStats } from './inspectText.js';
|
|
4
4
|
import { getRecommendedSanctifierOptions } from './inspectText.js';
|
|
5
5
|
import { summonSanctifier } from './sanctifyText.js';
|
|
6
|
+
import { sanctifyStrict } from './sanctifyText.js';
|
|
7
|
+
import { sanctifyLoose } from './sanctifyText.js';
|
|
8
|
+
import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
|
|
9
|
+
import { sanctifyKeyboardOnly } from './sanctifyText.js';
|
|
10
|
+
import { sanctifyPasted } from './sanctifyText.js';
|
|
6
11
|
import { revealInvisible } from './reveal.js';
|
|
7
|
-
|
|
12
|
+
import { VERSION } from './version.generated.js';
|
|
13
|
+
export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
|
|
8
14
|
export type UnicodeTrashReport = import('./inspectText.js').UnicodeTrashReport;
|
|
9
15
|
export type TrashCategory = import('./inspectText.js').TrashCategory;
|
|
10
16
|
export type TrashFinding = import('./inspectText.js').TrashFinding;
|
package/src/index.js
CHANGED
|
@@ -4,12 +4,15 @@
|
|
|
4
4
|
import { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions } from './inspectText.js';
|
|
5
5
|
export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions };
|
|
6
6
|
|
|
7
|
-
import { summonSanctifier } from './sanctifyText.js';
|
|
8
|
-
export { summonSanctifier };
|
|
7
|
+
import { summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted } from './sanctifyText.js';
|
|
8
|
+
export { summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted };
|
|
9
9
|
|
|
10
10
|
import { revealInvisible } from './reveal.js';
|
|
11
11
|
export { revealInvisible };
|
|
12
12
|
|
|
13
|
+
import { VERSION } from './version.generated.js';
|
|
14
|
+
export { VERSION };
|
|
15
|
+
|
|
13
16
|
// Type-only re-exports so TypeScript consumers can name these from the package root.
|
|
14
17
|
/** @typedef {import('./inspectText.js').UnicodeTrashReport} UnicodeTrashReport */
|
|
15
18
|
/** @typedef {import('./inspectText.js').TrashCategory} TrashCategory */
|
package/src/inspectText.js
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
// ./src/inspectText.js
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
INVISIBLE_TRASH_REGEX,
|
|
6
|
-
EMOJI_REGEX,
|
|
7
|
-
ASCII_KEYBOARD_SAFE_REGEX,
|
|
8
|
-
normalizeTypographicJank,
|
|
9
|
-
} from './sanctifyText.js';
|
|
10
|
-
import { classifyCodeUnit, classifyAstral } from './categories.generated.js';
|
|
3
|
+
import { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX, classifyCodeUnit, classifyAstral } from './categories.generated.js';
|
|
4
|
+
import { EMOJI_REGEX, ASCII_KEYBOARD_SAFE_REGEX, normalizeTypographicJank } from './textPatterns.js';
|
|
11
5
|
|
|
12
6
|
/** @typedef {import('./sanctifyText.js').SanctifyOptions} SanctifyOptions */
|
|
13
7
|
|
package/src/sanctifyText.js
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX } from './categories.generated.js';
|
|
4
4
|
export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX };
|
|
5
|
+
import { ASCII_KEYBOARD_SAFE_REGEX, EMOJI_REGEX, normalizeTypographicJank } from './textPatterns.js';
|
|
6
|
+
export { ASCII_KEYBOARD_SAFE_REGEX, EMOJI_REGEX, normalizeTypographicJank };
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
/**
|
|
@@ -45,7 +47,7 @@ export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX };
|
|
|
45
47
|
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
46
48
|
* @returns {(text: string) => string}
|
|
47
49
|
*/
|
|
48
|
-
|
|
50
|
+
function createSanctifier(defaultOptions = {}) {
|
|
49
51
|
const normalizeSpaces = !!defaultOptions.normalizeSpaces;
|
|
50
52
|
const purgeZeroWidthSpaces = !!defaultOptions.purgeZeroWidthSpaces;
|
|
51
53
|
const purgeInvisibleChars = !!defaultOptions.purgeInvisibleChars;
|
|
@@ -91,7 +93,10 @@ export function summonSanctifier(defaultOptions = {}) {
|
|
|
91
93
|
);
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
// ---
|
|
96
|
+
// --- Presets ---
|
|
97
|
+
//
|
|
98
|
+
// Each preset is its own export, so a bundle that imports one preset leaves out the
|
|
99
|
+
// others. summonSanctifier.strict and the rest are the same functions.
|
|
95
100
|
|
|
96
101
|
/**
|
|
97
102
|
* Strict sanitizer:
|
|
@@ -99,21 +104,24 @@ export function summonSanctifier(defaultOptions = {}) {
|
|
|
99
104
|
* - Collapse all newlines
|
|
100
105
|
* - Collapse spaces
|
|
101
106
|
* - Nuke control characters
|
|
107
|
+
* @param {string} text
|
|
108
|
+
* @returns {string}
|
|
102
109
|
*/
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
);
|
|
110
|
+
export function sanctifyStrict(text) {
|
|
111
|
+
return sanctifyText(
|
|
112
|
+
text,
|
|
113
|
+
true, // purgeInvisibleChars
|
|
114
|
+
true, // purgeEmojis
|
|
115
|
+
true, // nukeControls
|
|
116
|
+
false, // keyboardOnlyFilter
|
|
117
|
+
true, // normalizeNewlines
|
|
118
|
+
true, // trimSpacesAroundNewlines
|
|
119
|
+
true, // collapseNewLines
|
|
120
|
+
false, // preserveParagraphs
|
|
121
|
+
true, // collapseSpaces
|
|
122
|
+
true // finalTrim
|
|
123
|
+
);
|
|
124
|
+
}
|
|
117
125
|
|
|
118
126
|
|
|
119
127
|
/**
|
|
@@ -121,21 +129,24 @@ summonSanctifier.strict = text => sanctifyText(
|
|
|
121
129
|
* - Collapse spaces
|
|
122
130
|
* - Preserve paragraphs
|
|
123
131
|
* - Normalize newlines
|
|
132
|
+
* @param {string} text
|
|
133
|
+
* @returns {string}
|
|
124
134
|
*/
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
);
|
|
135
|
+
export function sanctifyLoose(text) {
|
|
136
|
+
return sanctifyText(
|
|
137
|
+
text,
|
|
138
|
+
false, // purgeInvisibleChars
|
|
139
|
+
false, // purgeEmojis
|
|
140
|
+
false, // nukeControls
|
|
141
|
+
false, // keyboardOnlyFilter
|
|
142
|
+
true, // normalizeNewlines
|
|
143
|
+
true, // trimSpacesAroundNewlines
|
|
144
|
+
true, // collapseNewLines
|
|
145
|
+
true, // preserveParagraphs
|
|
146
|
+
true, // collapseSpaces
|
|
147
|
+
true // finalTrim
|
|
148
|
+
);
|
|
149
|
+
}
|
|
139
150
|
|
|
140
151
|
|
|
141
152
|
/**
|
|
@@ -143,21 +154,24 @@ summonSanctifier.loose = text => sanctifyText(
|
|
|
143
154
|
* - Keeps emojis and printable ASCII
|
|
144
155
|
* - Strips non-standard characters
|
|
145
156
|
* - Normalizes typographic trash
|
|
157
|
+
* @param {string} text
|
|
158
|
+
* @returns {string}
|
|
146
159
|
*/
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
);
|
|
160
|
+
export function sanctifyKeyboardOnlyEmoji(text) {
|
|
161
|
+
return sanctifyText(
|
|
162
|
+
text,
|
|
163
|
+
false, // purgeInvisibleChars
|
|
164
|
+
false, // purgeEmojis
|
|
165
|
+
false, // nukeControls
|
|
166
|
+
true, // keyboardOnlyFilter
|
|
167
|
+
true, // normalizeNewlines
|
|
168
|
+
true, // trimSpacesAroundNewlines
|
|
169
|
+
false, // collapseNewLines
|
|
170
|
+
false, // preserveParagraphs
|
|
171
|
+
false, // collapseSpaces
|
|
172
|
+
true // finalTrim
|
|
173
|
+
);
|
|
174
|
+
}
|
|
161
175
|
|
|
162
176
|
|
|
163
177
|
/**
|
|
@@ -165,21 +179,24 @@ summonSanctifier.keyboardOnlyEmoji = text => sanctifyText(
|
|
|
165
179
|
* - Removes emojis
|
|
166
180
|
* - Collapses all whitespace
|
|
167
181
|
* - Restricts to printable ASCII only
|
|
182
|
+
* @param {string} text
|
|
183
|
+
* @returns {string}
|
|
168
184
|
*/
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
);
|
|
185
|
+
export function sanctifyKeyboardOnly(text) {
|
|
186
|
+
return sanctifyText(
|
|
187
|
+
text,
|
|
188
|
+
true, // purgeInvisibleChars
|
|
189
|
+
true, // purgeEmojis
|
|
190
|
+
true, // nukeControls
|
|
191
|
+
true, // keyboardOnlyFilter
|
|
192
|
+
true, // normalizeNewlines
|
|
193
|
+
true, // trimSpacesAroundNewlines
|
|
194
|
+
true, // collapseNewLines
|
|
195
|
+
false, // preserveParagraphs
|
|
196
|
+
true, // collapseSpaces
|
|
197
|
+
true // finalTrim
|
|
198
|
+
);
|
|
199
|
+
}
|
|
183
200
|
|
|
184
201
|
|
|
185
202
|
/**
|
|
@@ -189,25 +206,70 @@ summonSanctifier.keyboardOnly = text => sanctifyText(
|
|
|
189
206
|
* - Zero-width spaces, word joiners and BOMs are removed
|
|
190
207
|
* - Newlines are normalized to `\n`
|
|
191
208
|
* Keeps emoji joiners, variation selectors, bidi marks, indentation and line breaks.
|
|
209
|
+
* @param {string} text
|
|
210
|
+
* @returns {string}
|
|
192
211
|
*/
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
);
|
|
212
|
+
export function sanctifyPasted(text) {
|
|
213
|
+
return sanctifyText(
|
|
214
|
+
text,
|
|
215
|
+
false, // purgeInvisibleChars
|
|
216
|
+
false, // purgeEmojis
|
|
217
|
+
false, // nukeControls
|
|
218
|
+
false, // keyboardOnlyFilter
|
|
219
|
+
true, // normalizeNewlines
|
|
220
|
+
false, // trimSpacesAroundNewlines
|
|
221
|
+
false, // collapseNewLines
|
|
222
|
+
false, // preserveParagraphs
|
|
223
|
+
false, // collapseSpaces
|
|
224
|
+
false, // finalTrim
|
|
225
|
+
Infinity, // maxLength
|
|
226
|
+
'throw', // onMaxLength
|
|
227
|
+
true, // normalizeSpaces
|
|
228
|
+
true // purgeZeroWidthSpaces
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Summons a customized sanctifier function with pre-bound booleans.
|
|
235
|
+
*
|
|
236
|
+
* Accepts full flag names and returns a text-cleaning function.
|
|
237
|
+
*
|
|
238
|
+
* @param {Object} [defaultOptions={}]
|
|
239
|
+
* @param {boolean} [defaultOptions.purgeInvisibleChars]
|
|
240
|
+
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
241
|
+
* @param {boolean} [defaultOptions.nukeControls]
|
|
242
|
+
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
243
|
+
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
244
|
+
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
245
|
+
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
246
|
+
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
247
|
+
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
248
|
+
* @param {boolean} [defaultOptions.finalTrim]
|
|
249
|
+
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
250
|
+
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
251
|
+
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
252
|
+
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
253
|
+
* @returns {(text: string) => string}
|
|
254
|
+
*
|
|
255
|
+
* The presets are attached as properties (`summonSanctifier.strict`, `.loose`,
|
|
256
|
+
* `.keyboardOnlyEmoji`, `.keyboardOnly`, `.pasted`) and are also standalone exports.
|
|
257
|
+
* Built in a call marked pure, so a bundle that imports only a preset leaves it out.
|
|
258
|
+
* @type {typeof createSanctifier & {
|
|
259
|
+
* strict: typeof sanctifyStrict,
|
|
260
|
+
* loose: typeof sanctifyLoose,
|
|
261
|
+
* keyboardOnlyEmoji: typeof sanctifyKeyboardOnlyEmoji,
|
|
262
|
+
* keyboardOnly: typeof sanctifyKeyboardOnly,
|
|
263
|
+
* pasted: typeof sanctifyPasted,
|
|
264
|
+
* }}
|
|
265
|
+
*/
|
|
266
|
+
export const summonSanctifier = /* @__PURE__ */ Object.assign(createSanctifier, {
|
|
267
|
+
strict: sanctifyStrict,
|
|
268
|
+
loose: sanctifyLoose,
|
|
269
|
+
keyboardOnlyEmoji: sanctifyKeyboardOnlyEmoji,
|
|
270
|
+
keyboardOnly: sanctifyKeyboardOnly,
|
|
271
|
+
pasted: sanctifyPasted,
|
|
272
|
+
});
|
|
211
273
|
|
|
212
274
|
|
|
213
275
|
/**
|
|
@@ -374,13 +436,6 @@ function extractEmojiSequences(text) {
|
|
|
374
436
|
return out;
|
|
375
437
|
}
|
|
376
438
|
|
|
377
|
-
/**
|
|
378
|
-
* Matches any character that is NOT:
|
|
379
|
-
* - Printable ASCII (U+0020–U+007E)
|
|
380
|
-
* - Newline characters: \n (LF), \r (CR)
|
|
381
|
-
* This allows for \n, \r, and \r\n line endings.
|
|
382
|
-
*/
|
|
383
|
-
export const ASCII_KEYBOARD_SAFE_REGEX = /[^\x20-\x7E\r\n]+/gu;
|
|
384
439
|
|
|
385
440
|
/**
|
|
386
441
|
* Removes all non-keyboard characters, preserving:
|
|
@@ -402,64 +457,6 @@ function purgeNonKeyboardChars(text, purgeEmojis = false) {
|
|
|
402
457
|
}
|
|
403
458
|
|
|
404
459
|
|
|
405
|
-
// Smart apostrophes (single quote variations)
|
|
406
|
-
const SMART_SINGLE_QUOTES_REGEX = /[\u2018\u2019\u201A\u201B\u2032\u2035]/g;
|
|
407
|
-
|
|
408
|
-
// Smart double quotes (including primes and guillemets)
|
|
409
|
-
const SMART_DOUBLE_QUOTES_REGEX = /[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g;
|
|
410
|
-
|
|
411
|
-
// Dash variants: en dash, em dash, horizontal bar, figure dash, minus
|
|
412
|
-
const UNICODE_DASHES_REGEX = /[\u2012\u2013\u2014\u2015\u2212]/g;
|
|
413
|
-
|
|
414
|
-
// Ellipsis character
|
|
415
|
-
const ELLIPSIS_REGEX = /\u2026/g;
|
|
416
|
-
|
|
417
|
-
// Bullets and middle dot
|
|
418
|
-
const BULLETS_REGEX = /[\u2022\u00B7]/g;
|
|
419
|
-
|
|
420
|
-
// Full-width ASCII punctuation: U+FF01 - U+FF5E
|
|
421
|
-
const FULLWIDTH_PUNCTUATION_REGEX = /[\uFF01-\uFF5E]/g;
|
|
422
|
-
|
|
423
|
-
/**
|
|
424
|
-
* Normalizes typographic Unicode punctuation into ASCII equivalents.
|
|
425
|
-
* @param {string} text
|
|
426
|
-
* @returns {string}
|
|
427
|
-
*/
|
|
428
|
-
export function normalizeTypographicJank(text) {
|
|
429
|
-
return text
|
|
430
|
-
.replace(SMART_SINGLE_QUOTES_REGEX, "'")
|
|
431
|
-
.replace(SMART_DOUBLE_QUOTES_REGEX, '"')
|
|
432
|
-
.replace(UNICODE_DASHES_REGEX, '-')
|
|
433
|
-
.replace(ELLIPSIS_REGEX, '...')
|
|
434
|
-
.replace(BULLETS_REGEX, '*')
|
|
435
|
-
.replace(FULLWIDTH_PUNCTUATION_REGEX, m =>
|
|
436
|
-
String.fromCharCode(m.charCodeAt(0) - 0xFEE0)
|
|
437
|
-
);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
/** @type {RegExp} */
|
|
442
|
-
export let EMOJI_REGEX;
|
|
443
|
-
|
|
444
|
-
/**
|
|
445
|
-
* Try Unicode property escape regex (preferred).
|
|
446
|
-
* Fallback to basic emoji ranges if unsupported.
|
|
447
|
-
*/
|
|
448
|
-
try {
|
|
449
|
-
EMOJI_REGEX = new RegExp(
|
|
450
|
-
'(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)',
|
|
451
|
-
'gu'
|
|
452
|
-
);
|
|
453
|
-
} catch {
|
|
454
|
-
// Fallback: wide-range emoji component match (flags, tones, symbols)
|
|
455
|
-
// * Covers:
|
|
456
|
-
// * - Emoji base chars (1F300–1FAFF)
|
|
457
|
-
// * - Dingbats (2700–27BF) like ❌, ✅, ☑️
|
|
458
|
-
// * - Skin tones (1F3FB–1F3FF)
|
|
459
|
-
// * - ZWJ (200D), variation selectors (FE0F)
|
|
460
|
-
// * - Regional indicators (1F1E6–1F1FF)
|
|
461
|
-
EMOJI_REGEX = /[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu;
|
|
462
|
-
}
|
|
463
460
|
|
|
464
461
|
/**
|
|
465
462
|
* Removes all emoji characters using Unicode property escapes.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// src/textPatterns.js
|
|
2
|
+
//
|
|
3
|
+
// Patterns shared by the inspector and the sanitizer. They live apart from
|
|
4
|
+
// sanctifyText.js so that importing an inspector does not pull in the sanitizer
|
|
5
|
+
// engine and its presets (see the size probes in scripts/check-size.js).
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Matches any character that is NOT:
|
|
10
|
+
* - Printable ASCII (U+0020–U+007E)
|
|
11
|
+
* - Newline characters: \n (LF), \r (CR)
|
|
12
|
+
* This allows for \n, \r, and \r\n line endings.
|
|
13
|
+
*/
|
|
14
|
+
export const ASCII_KEYBOARD_SAFE_REGEX = /[^\x20-\x7E\r\n]+/gu;
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
// Smart apostrophes (single quote variations)
|
|
18
|
+
const SMART_SINGLE_QUOTES_REGEX = /[\u2018\u2019\u201A\u201B\u2032\u2035]/g;
|
|
19
|
+
|
|
20
|
+
// Smart double quotes (including primes and guillemets)
|
|
21
|
+
const SMART_DOUBLE_QUOTES_REGEX = /[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g;
|
|
22
|
+
|
|
23
|
+
// Dash variants: en dash, em dash, horizontal bar, figure dash, minus
|
|
24
|
+
const UNICODE_DASHES_REGEX = /[\u2012\u2013\u2014\u2015\u2212]/g;
|
|
25
|
+
|
|
26
|
+
// Ellipsis character
|
|
27
|
+
const ELLIPSIS_REGEX = /\u2026/g;
|
|
28
|
+
|
|
29
|
+
// Bullets and middle dot
|
|
30
|
+
const BULLETS_REGEX = /[\u2022\u00B7]/g;
|
|
31
|
+
|
|
32
|
+
// Full-width ASCII punctuation: U+FF01 - U+FF5E
|
|
33
|
+
const FULLWIDTH_PUNCTUATION_REGEX = /[\uFF01-\uFF5E]/g;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Normalizes typographic Unicode punctuation into ASCII equivalents.
|
|
37
|
+
* @param {string} text
|
|
38
|
+
* @returns {string}
|
|
39
|
+
*/
|
|
40
|
+
export function normalizeTypographicJank(text) {
|
|
41
|
+
return text
|
|
42
|
+
.replace(SMART_SINGLE_QUOTES_REGEX, "'")
|
|
43
|
+
.replace(SMART_DOUBLE_QUOTES_REGEX, '"')
|
|
44
|
+
.replace(UNICODE_DASHES_REGEX, '-')
|
|
45
|
+
.replace(ELLIPSIS_REGEX, '...')
|
|
46
|
+
.replace(BULLETS_REGEX, '*')
|
|
47
|
+
.replace(FULLWIDTH_PUNCTUATION_REGEX, m =>
|
|
48
|
+
String.fromCharCode(m.charCodeAt(0) - 0xFEE0)
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Try Unicode property escape regex (preferred).
|
|
55
|
+
* Fallback to basic emoji ranges if unsupported.
|
|
56
|
+
* Built in a function marked pure, so bundlers drop it when nothing uses it.
|
|
57
|
+
* @returns {RegExp}
|
|
58
|
+
*/
|
|
59
|
+
function buildEmojiRegex() {
|
|
60
|
+
try {
|
|
61
|
+
return new RegExp(
|
|
62
|
+
'(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)',
|
|
63
|
+
'gu'
|
|
64
|
+
);
|
|
65
|
+
} catch {
|
|
66
|
+
// Fallback: wide-range emoji component match (flags, tones, symbols)
|
|
67
|
+
// * Covers:
|
|
68
|
+
// * - Emoji base chars (1F300–1FAFF)
|
|
69
|
+
// * - Dingbats (2700–27BF) like ❌, ✅, ☑
|
|
70
|
+
// * - Skin tones (1F3FB–1F3FF)
|
|
71
|
+
// * - ZWJ (200D), variation selectors (FE0F)
|
|
72
|
+
// * - Regional indicators (1F1E6–1F1FF)
|
|
73
|
+
return /[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** @type {RegExp} */
|
|
78
|
+
export const EMOJI_REGEX = /* @__PURE__ */ buildEmojiRegex();
|