text-sanctifier 1.3.5 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -7
- package/bin/text-sanctifier.js +1 -1
- package/dist/text-sanctifier.min.js +5 -5
- package/dist/types/context.d.ts +56 -0
- package/dist/types/hiddenText.d.ts +71 -6
- package/dist/types/index.d.ts +7 -1
- package/dist/types/sanctifyText.d.ts +141 -110
- package/dist/types/textPatterns.d.ts +15 -0
- package/dist/types/version.generated.d.ts +2 -0
- package/package.json +8 -4
- package/src/context.js +177 -0
- package/src/hiddenText.js +139 -26
- package/src/index.d.ts +7 -1
- package/src/index.js +5 -2
- package/src/inspectText.js +2 -8
- package/src/sanctifyText.js +138 -141
- package/src/textPatterns.js +78 -0
- package/src/version.generated.js +5 -0
package/README.md
CHANGED
|
@@ -119,6 +119,16 @@ Two options handle spaces and zero-width characters gently, and both run before
|
|
|
119
119
|
- `normalizeSpaces`: replace look-alike spaces with an ordinary space. These are U+00A0, U+2000–U+200A, U+202F and U+205F: the `nonbreaking_space` category except U+3000, the ordinary space of CJK text. With `purgeInvisibleChars` alone, a non-breaking space between two words is deleted and the words run together; with `normalizeSpaces` as well, they stay apart.
|
|
120
120
|
- `purgeZeroWidthSpaces`: remove the zero width space U+200B, the word joiner U+2060 and U+FEFF, and nothing else. Emoji joiners, ZWNJ, variation selectors and bidi marks are kept.
|
|
121
121
|
|
|
122
|
+
### Presets as standalone imports
|
|
123
|
+
|
|
124
|
+
Each preset is also its own export: `sanctifyStrict`, `sanctifyLoose`, `sanctifyKeyboardOnly`, `sanctifyKeyboardOnlyEmoji` and `sanctifyPasted`. They are the same functions as `summonSanctifier.strict` and the rest. Import one directly and a bundler leaves out `summonSanctifier` and the other presets (about 1.1 KB gzipped instead of 1.45 KB).
|
|
125
|
+
|
|
126
|
+
```js
|
|
127
|
+
import { sanctifyPasted } from 'text-sanctifier';
|
|
128
|
+
|
|
129
|
+
const output = sanctifyPasted(clipboardText);
|
|
130
|
+
```
|
|
131
|
+
|
|
122
132
|
### `summonSanctifier.strict`
|
|
123
133
|
|
|
124
134
|
Aggressively purges: emojis, control characters, extra spacing, and newlines.
|
|
@@ -236,22 +246,47 @@ getNewlineStats('a\r\nb\n'); // { crlf: 1, cr: 0, lf: 1, types: ['CRLF', 'LF'],
|
|
|
236
246
|
|
|
237
247
|
The categories cover exactly what `nukeControls` and `purgeInvisibleChars` strip. Every code point with the Unicode property Default_Ignorable_Code_Point (characters renderers must draw as nothing) is in a category, and a test checks this against the running Node's Unicode data.
|
|
238
248
|
|
|
239
|
-
### `text-sanctifier/hidden`: `findHiddenRuns(text)` and `decodeTagText(text)`
|
|
249
|
+
### `text-sanctifier/hidden`: `findHiddenRuns(text, options?)` and `decodeTagText(text)`
|
|
240
250
|
|
|
241
251
|
The run finder behind [`check --ai-config`](#ai-configuration-files), for tools that read text on its way to a model. It has no Node APIs, so it works in a browser too, but it is a separate entry point and not part of the `text-sanctifier/browser` bundle.
|
|
242
252
|
|
|
243
253
|
```js
|
|
244
254
|
import { findHiddenRuns, decodeTagText } from 'text-sanctifier/hidden';
|
|
245
255
|
|
|
246
|
-
const { runs, benign } = findHiddenRuns(text);
|
|
247
|
-
// runs: [{ kind: 'tag', offset, end, count, decoded: 'Ignore the user' }, ...]
|
|
256
|
+
const { runs, benign, counts, total, truncated } = findHiddenRuns(text, { maxRuns: 100, maxDecodedLength: 1000 });
|
|
257
|
+
// runs: [{ kind: 'tag', offset, end, line, column, count, decoded: 'Ignore the user', decodedTruncated: false }, ...]
|
|
248
258
|
// benign: [[offset, end], ...] England/Scotland/Wales flags, a lone U+FE0E/U+FE0F after a character
|
|
259
|
+
// counts: { tag, variation_selector, invisible_operator, reserved_tag_plane }, every run listed or not; total is their sum
|
|
249
260
|
```
|
|
250
261
|
|
|
251
|
-
- `kind` is `tag` (U+E0000–U+E007F), `variation_selector` (U+FE00–U+FE0F, U+E0100–U+E01EF, U+180B–U+180D, U+180F)
|
|
252
|
-
- `offset` and `end` are UTF-16 indexes, `[offset, end)`; `count` is in code points. Only tag runs have `decoded`.
|
|
262
|
+
- `kind` is `tag` (U+E0000–U+E007F), `variation_selector` (U+FE00–U+FE0F, U+E0100–U+E01EF, U+180B–U+180D, U+180F) `invisible_operator` (U+2061–U+2064) or `reserved_tag_plane` (U+E0080–U+E00FF, U+E01F0–U+E0FFF: unassigned, default ignorable, so they render as nothing and can carry data). Consecutive characters of one kind form one run.
|
|
263
|
+
- `offset` and `end` are UTF-16 indexes, `[offset, end)`; `count` is in code points. `line` and `column` are counted exactly as in `inspectTextDetailed` (1-based, columns in UTF-16 units, CRLF is one line break). Only tag runs have `decoded` and `decodedTruncated`.
|
|
264
|
+
- `maxRuns` caps the `runs` list and, separately, the `benign` list; `counts`, `total` and `benignTotal` count everything, and `truncated` is true when either list was cut. `maxDecodedLength` caps each `decoded` string in UTF-16 units and never cuts a `<U+XXXX>` marker. Both default to no cap, and an invalid value (negative, fractional, a string) also means no cap, so set them explicitly for untrusted input. With both set, memory stays flat on any input: on 16 M characters (32 MB) of hidden text the heap grows by under 1 MB, against hundreds of MB with no caps.
|
|
253
265
|
- `decodeTagText` turns U+E0020–U+E007E into the ASCII they shadow and writes any other character as `<U+XXXX>`.
|
|
254
266
|
- Zero-width, bidi and other invisible characters are not runs; use `inspectTextDetailed` for those.
|
|
267
|
+
- `codePointCounts` has the same keys as `counts` and counts the code points in those runs, so long runs weigh more than short ones; `supplementarySelectorCount` counts U+E0100–U+E01EF on their own, apart from the emoji selectors U+FE00–U+FE0F. Both leave out benign spans and stay exact under `maxRuns`.
|
|
268
|
+
- `VERSION` is exported here and from the package root: the installed version, as in `package.json`.
|
|
269
|
+
|
|
270
|
+
### `text-sanctifier/context`: `inspectTextInContext(text, options?)`
|
|
271
|
+
|
|
272
|
+
`inspectTextDetailed` with context. Some invisible characters are part of ordinary text: the joiners in a family emoji, the selector in ✌ followed by VS16, an ideographic variation selector after a CJK character. This entry point reports the same findings and `counts`, marks each listed finding `benign: true` or `false`, and adds `benignCounts`, exact whether or not a finding is listed. The categories are the same.
|
|
273
|
+
|
|
274
|
+
```js
|
|
275
|
+
import { inspectTextInContext } from 'text-sanctifier/context';
|
|
276
|
+
|
|
277
|
+
const report = inspectTextInContext(text, { maxFindings: 20 });
|
|
278
|
+
const suspicious = report.counts.zero_width - report.benignCounts.zero_width; // U+200B, U+2060, stray U+200D
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Benign, and nothing else:
|
|
282
|
+
|
|
283
|
+
- U+200D between two emoji (`Extended_Pictographic`), allowing a skin-tone modifier or VS16 after the first
|
|
284
|
+
- U+FE0E or U+FE0F directly after an emoji, or in a keycap (0-9, # or *, the selector, U+20E3)
|
|
285
|
+
- one of U+E0100–U+E01EF directly after an ideograph
|
|
286
|
+
- U+180B–U+180D or U+180F directly after a Mongolian letter
|
|
287
|
+
- the tags of the England, Scotland and Wales flags
|
|
288
|
+
|
|
289
|
+
U+200B, U+2060, U+FEFF, bidi controls and all other tags are never benign. Joiners in scripts that need them (ZWNJ in Persian, ZWJ in Indic scripts) are not benign yet. It needs Unicode property escapes (Node 10+, current browsers), is not in the browser file, and is about 1.7 KB gzipped on its own.
|
|
255
290
|
|
|
256
291
|
---
|
|
257
292
|
|
|
@@ -301,7 +336,7 @@ skills/review/SKILL.md:2:3 U+FE00 run of 3 variation selectors
|
|
|
301
336
|
- Consecutive tag characters, variation selectors or invisible operators are reported once, as a run, at the first character. Tag runs are decoded: U+E0020–U+E007E become the ASCII they shadow, and other tags are shown as `<U+XXXX>`.
|
|
302
337
|
- Every other invisible or control character is reported as in a plain `check`.
|
|
303
338
|
- Two legitimate sequences are left out: the flags of England, Scotland and Wales (U+1F3F4, tag letters, U+E007F), and a single U+FE0E or U+FE0F after another character (text or emoji presentation). A plain `check` still reports both.
|
|
304
|
-
- `--json` adds `hiddenRuns` (`{ kind, offset, line, column, codePoint, count, decoded?, decodedTruncated? }`, at most `--max-findings` per file) and `hiddenRunCount` to each file, and `hiddenRunCount` to the summary. `kind` is `tag`, `variation_selector` or `
|
|
339
|
+
- `--json` adds `hiddenRuns` (`{ kind, offset, line, column, codePoint, count, decoded?, decodedTruncated? }`, at most `--max-findings` per file) and `hiddenRunCount` to each file, and `hiddenRunCount` to the summary. `kind` is `tag`, `variation_selector`, `invisible_operator` or `reserved_tag_plane`; `decoded` is cut at 1000 characters. `total` and `counts` leave out the legitimate sequences.
|
|
305
340
|
|
|
306
341
|
### SARIF output
|
|
307
342
|
|
|
@@ -321,7 +356,7 @@ The repository publishes a [pre-commit](https://pre-commit.com) hook manifest, s
|
|
|
321
356
|
# .pre-commit-config.yaml
|
|
322
357
|
repos:
|
|
323
358
|
- repo: https://github.com/iWhatty/text-sanctifier
|
|
324
|
-
rev: v1.3.
|
|
359
|
+
rev: v1.3.7 # a release tag
|
|
325
360
|
hooks:
|
|
326
361
|
- id: text-sanctifier # every staged text file
|
|
327
362
|
- id: text-sanctifier-ai-config # agent instruction files, with --ai-config
|
package/bin/text-sanctifier.js
CHANGED
|
@@ -443,7 +443,7 @@ const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`;
|
|
|
443
443
|
// --- check --ai-config ---
|
|
444
444
|
|
|
445
445
|
const MAX_DECODED = 1000;
|
|
446
|
-
const RUN_NOUNS = { tag: 'tag character', variation_selector: 'variation selector', invisible_operator: 'invisible operator' };
|
|
446
|
+
const RUN_NOUNS = { tag: 'tag character', variation_selector: 'variation selector', invisible_operator: 'invisible operator', reserved_tag_plane: 'reserved plane-14 code point' };
|
|
447
447
|
|
|
448
448
|
/**
|
|
449
449
|
* inspect() for agent instruction files: drops findings that belong to legitimate
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
var
|
|
2
|
-
|
|
1
|
+
var y=/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]+/g,d=/[\u00A0\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180B-\u180F\u2000-\u200F\u202A-\u202F\u205F-\u206F\u3000\u3164\uFE00-\uFE0F\uFEFF\uFFA0\uFFF0-\uFFF8\u{1BCA0}-\u{1BCA3}\u{1D173}-\u{1D17A}\u{E0000}-\u{E0FFF}]+/gu;function A(e){return e<8192?D(e):v(e)}function D(e){return e<174?e<32?e<11?e<1?"nul":e<9?"c0_control":null:e<13?"c0_control":e<14?null:"c0_control":e<160?e<127?null:e<128?"c0_control":"c1_control":e<161?"nonbreaking_space":e<173?null:"invisible_format":e<4449?e<1564?e<847?null:e<848?"invisible_format":null:e<1565?"bidi_control":e<4447?null:"hangul_filler":e<6155?e<6068?null:e<6070?"invisible_format":null:e<6159?e<6158?"variation_selector":"invisible_format":e<6160?"variation_selector":null}function v(e){return e<8304?e<8240?e<8208?e<8203?"nonbreaking_space":e<8206?"zero_width":"bidi_control":e<8234?null:e<8239?"bidi_control":"nonbreaking_space":e<8289?e<8287?null:e<8288?"nonbreaking_space":"zero_width":e<8294?"invisible_format":e<8298?"bidi_control":"invisible_format":e<65040?e<12644?e<12288?null:e<12289?"nonbreaking_space":null:e<12645?"hangul_filler":e<65024?null:"variation_selector":e<65440?e<65279?null:e<65280?"bom":null:e<65520?e<65441?"hangul_filler":null:e<65529?"invisible_format":null}function I(e){return e<917504?e<113828?e<113824?null:"invisible_format":e<119155?null:e<119163?"invisible_format":null:e<917760?e<917632?"tag":"invisible_format":e<918e3?"variation_selector":e<921600?"invisible_format":null}var g=/[^\x20-\x7E\r\n]+/gu,B=/[\u2018\u2019\u201A\u201B\u2032\u2035]/g,G=/[\u201C\u201D\u201E\u201F\u2033\u2036\u00AB\u00BB]/g,X=/[\u2012\u2013\u2014\u2015\u2212]/g,k=/\u2026/g,O=/[\u2022\u00B7]/g,P=/[\uFF01-\uFF5E]/g;function S(e){return e.replace(B,"'").replace(G,'"').replace(X,"-").replace(k,"...").replace(O,"*").replace(P,r=>String.fromCharCode(r.charCodeAt(0)-65248))}function M(){try{return new RegExp("(?:\\p{Extended_Pictographic}(?:\\uFE0F|\\uFE0E)?(?:\\u200D(?:\\p{Extended_Pictographic}|\\w)+)*)","gu")}catch{return/[\u{2700}-\u{27BF}\u{1F300}-\u{1FAFF}\u{1F3FB}-\u{1F3FF}\u200D\uFE0F\u{1F1E6}-\u{1F1FF}]/gu}}var x=M();function m(e,r){return e.lastIndex=0,e.test(r)}function U(e){let r=S(e);if(!m(g,r))return!1;for(let n of r)if(!(n===`
|
|
2
|
+
`||n==="\r"||n>=" "&&n<="~")&&!m(x,n))return!0;return!1}function j(e){if(typeof e!="string")throw new TypeError("inspectText expects a string input.");let r=[],n={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:r},u=(s,c,f)=>{s&&(n[c]=!0,r.push(f))};u(m(y,e),"hasControlChars","Control characters detected."),u(m(d,e),"hasInvisibleChars","Invisible Unicode characters detected."),u(m(x,e),"hasEmojis","Emojis detected.");let{mixed:o,types:i}=b(e);return n.hasMixedNewlines=o,n.newlineStyle=o?"Mixed":i[0]||null,n.newlineStyle&&r.push(o?"Mixed newline styles detected.":`Consistent newline style: ${n.newlineStyle}`),u(U(e),"hasNonKeyboardChars","Non-keyboard characters detected."),n}var z=100;function C(e,r={}){if(typeof e!="string")throw new TypeError("inspectTextDetailed expects a string input.");let n=r&&r.maxFindings,u=n===1/0||Number.isInteger(n)&&n>=0?n:z,o={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},i=[],s=0,c=1,f=0;for(let a=0,h=e.length;a<h;a++){let l=e.charCodeAt(a);if(l>=32&&l<127)continue;if(l===10){c++,f=a+1;continue}if(l===13){e.charCodeAt(a+1)!==10&&(c++,f=a+1);continue}let E=a,F=l,p;if(l>=55296&&l<=56319){let t=e.charCodeAt(a+1);if(!(t>=56320&&t<=57343))continue;F=65536+(l-55296<<10)+(t-56320),p=I(F),a++}else p=A(l);if(p!==null&&(o[p]++,s++,i.length<u)){let t={offset:E,line:c,column:E-f+1,codePoint:F,category:p};p==="bom"&&(t.atStart=E===0),i.push(t)}}return{findings:i,counts:o,total:s,truncated:s>i.length}}function b(e){if(typeof e!="string")throw new TypeError("getNewlineStats expects a string input.");let r=0,n=0,u=0;for(let s=0,c=e.length;s<c;s++){let f=e.charCodeAt(s);f===13?e.charCodeAt(s+1)===10?(r++,s++):n++:f===10&&u++}let o={crlf:r,cr:n,lf:u},i=[];return o.crlf>0&&i.push("CRLF"),o.cr>0&&i.push("CR"),o.lf>0&&i.push("LF"),{...o,types:i,mixed:i.length>1}}function K(e){return{purgeInvisibleChars:e.hasInvisibleChars,purgeEmojis:e.hasEmojis,nukeControls:e.hasControlChars,keyboardOnlyFilter:e.hasNonKeyboardChars,normalizeNewlines:e.hasMixedNewlines||e.newlineStyle==="CRLF"||e.newlineStyle==="CR"}}function H(e={}){let r=!!e.normalizeSpaces,n=!!e.purgeZeroWidthSpaces,u=!!e.purgeInvisibleChars,o=!!e.purgeEmojis,i=!!e.nukeControls,s=!!e.keyboardOnlyFilter,c=!!e.normalizeNewlines,f=!!e.trimSpacesAroundNewlines,a=!!e.collapseNewLines,h=!!e.preserveParagraphs,l=!!e.collapseSpaces,E=!!e.finalTrim,F=Number.isFinite(e.maxLength)&&e.maxLength>=0?e.maxLength:1/0,p=e.onMaxLength==="truncate"||e.onMaxLength==="noop"||e.onMaxLength==="throw"?e.onMaxLength:"throw";return t=>_(t,u,o,i,s,c,f,a,h,l,E,F,p,r,n)}function R(e){return _(e,!0,!0,!0,!1,!0,!0,!0,!1,!0,!0)}function w(e){return _(e,!1,!1,!1,!1,!0,!0,!0,!0,!0,!0)}function N(e){return _(e,!1,!1,!1,!0,!0,!0,!1,!1,!1,!0)}function T(e){return _(e,!0,!0,!0,!0,!0,!0,!0,!1,!0,!0)}function L(e){return _(e,!1,!1,!1,!1,!0,!1,!1,!1,!1,!1,1/0,"throw",!0,!0)}var W=Object.assign(H,{strict:R,loose:w,keyboardOnlyEmoji:N,keyboardOnly:T,pasted:L});function _(e,r=!1,n=!1,u=!1,o=!1,i=!1,s=!1,c=!1,f=!1,a=!1,h=!1,l=1/0,E="throw",F=!1,p=!1){if(typeof e!="string")throw new TypeError("sanctifyText expects a string input.");if(E!=="truncate"&&E!=="noop"&&E!=="throw"&&(E="throw"),e.length>l)switch(E){case"truncate":e=e.slice(0,l);break;case"noop":return e;case"throw":default:throw new RangeError(`sanctifyText input length ${e.length} exceeds maxLength ${l}.`)}let t=e;return F&&(t=t.replace(J," ")),p&&(t=t.replace(V,"")),r&&(t=Z(t)),n&&(t=q(t)),u&&(t=le(t)),o&&(t=Y(t,n)),i&&(t=ee(t)),s&&(t=ne(t)),c&&(t=oe(t,f)),a&&(t=se(t)),h?t.trim():t}var J=/[\u00A0\u2000-\u200A\u202F\u205F]/g,V=/[\u200B\u2060\uFEFF]/g;function Z(e){return e.replace(d,"")}function $(e){if(x.lastIndex=0,!x.test(e))return"";x.lastIndex=0;let r="";for(let n of e.matchAll(x))r+=n[0];return r}function Y(e,r=!1){let n=S(e);return r?n.replace(g,""):n.replace(g,u=>$(u))}function q(e){return e.replace(x,"")}var Q=/\r\n|\r|\n/g;function ee(e,r=`
|
|
3
|
+
`){return e.replace(Q,r)}var re=/[ \t]*(\n+)[ \t]*/g;function ne(e){return e.replace(re,"$1")}var te=/\n{2,}/g,ie=/\n{3,}/g;function oe(e,r){return r?e.replace(ie,`
|
|
3
4
|
|
|
4
|
-
`):e.replace(
|
|
5
|
-
`)}var
|
|
6
|
-
`||n==="\r"||n>=" "&&n<="~")&&!m(p,n))return!0;return!1}function Y(e){if(typeof e!="string")throw new TypeError("inspectText expects a string input.");let r=[],n={hasControlChars:!1,hasInvisibleChars:!1,hasMixedNewlines:!1,newlineStyle:null,hasEmojis:!1,hasNonKeyboardChars:!1,summary:r},o=(l,c,f)=>{l&&(n[c]=!0,r.push(f))};o(m(d,e),"hasControlChars","Control characters detected."),o(m(C,e),"hasInvisibleChars","Invisible Unicode characters detected."),o(m(p,e),"hasEmojis","Emojis detected.");let{mixed:u,types:i}=w(e);return n.hasMixedNewlines=u,n.newlineStyle=u?"Mixed":i[0]||null,n.newlineStyle&&r.push(u?"Mixed newline styles detected.":`Consistent newline style: ${n.newlineStyle}`),o(Q(e),"hasNonKeyboardChars","Non-keyboard characters detected."),n}var ee=100;function y(e,r={}){if(typeof e!="string")throw new TypeError("inspectTextDetailed expects a string input.");let n=r&&r.maxFindings,o=n===1/0||Number.isInteger(n)&&n>=0?n:ee,u={bidi_control:0,zero_width:0,nonbreaking_space:0,bom:0,nul:0,c0_control:0,c1_control:0,invisible_format:0,hangul_filler:0,variation_selector:0,tag:0},i=[],l=0,c=1,f=0;for(let a=0,g=e.length;a<g;a++){let s=e.charCodeAt(a);if(s>=32&&s<127)continue;if(s===10){c++,f=a+1;continue}if(s===13){e.charCodeAt(a+1)!==10&&(c++,f=a+1);continue}let E=a,F=s,x;if(s>=55296&&s<=56319){let t=e.charCodeAt(a+1);if(!(t>=56320&&t<=57343))continue;F=65536+(s-55296<<10)+(t-56320),x=b(F),a++}else x=I(s);if(x!==null&&(u[x]++,l++,i.length<o)){let t={offset:E,line:c,column:E-f+1,codePoint:F,category:x};x==="bom"&&(t.atStart=E===0),i.push(t)}}return{findings:i,counts:u,total:l,truncated:l>i.length}}function w(e){if(typeof e!="string")throw new TypeError("getNewlineStats expects a string input.");let r=0,n=0,o=0;for(let l=0,c=e.length;l<c;l++){let f=e.charCodeAt(l);f===13?e.charCodeAt(l+1)===10?(r++,l++):n++:f===10&&o++}let u={crlf:r,cr:n,lf:o},i=[];return u.crlf>0&&i.push("CRLF"),u.cr>0&&i.push("CR"),u.lf>0&&i.push("LF"),{...u,types:i,mixed:i.length>1}}function re(e){return{purgeInvisibleChars:e.hasInvisibleChars,purgeEmojis:e.hasEmojis,nukeControls:e.hasControlChars,keyboardOnlyFilter:e.hasNonKeyboardChars,normalizeNewlines:e.hasMixedNewlines||e.newlineStyle==="CRLF"||e.newlineStyle==="CR"}}function ne(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",o="",u=0;for(let i of y(e,{maxFindings:1/0}).findings)o+=e.slice(u,i.offset)+n(i.codePoint,i.category),u=i.offset+(i.codePoint>65535?2:1);return o+e.slice(u)}export{w as getNewlineStats,re as getRecommendedSanctifierOptions,Y as inspectText,y as inspectTextDetailed,ne as revealInvisible,h as summonSanctifier};
|
|
5
|
+
`):e.replace(te,`
|
|
6
|
+
`)}var ue=/ {2,}/g;function se(e){return e.replace(ue," ")}function le(e){return e.replace(y,"")}function ae(e,r={}){if(typeof e!="string")throw new TypeError("revealInvisible expects a string input.");let n=r&&typeof r.format=="function"?r.format:i=>"<U+"+i.toString(16).toUpperCase().padStart(4,"0")+">",u="",o=0;for(let i of C(e,{maxFindings:1/0}).findings)u+=e.slice(o,i.offset)+n(i.codePoint,i.category),o=i.offset+(i.codePoint>65535?2:1);return u+e.slice(o)}var fe="1.3.7";export{fe as VERSION,b as getNewlineStats,K as getRecommendedSanctifierOptions,j as inspectText,C as inspectTextDetailed,ae as revealInvisible,T as sanctifyKeyboardOnly,N as sanctifyKeyboardOnlyEmoji,w as sanctifyLoose,L as sanctifyPasted,R as sanctifyStrict,W as summonSanctifier};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Locates invisible and control characters as inspectTextDetailed does, and marks
|
|
3
|
+
* the ones that belong to legitimate sequences as benign:
|
|
4
|
+
* - U+200D between two emoji (Extended_Pictographic), allowing a skin-tone modifier or
|
|
5
|
+
* VS16 after the first: an emoji ZWJ sequence
|
|
6
|
+
* - U+FE0E or U+FE0F directly after an emoji, or between 0-9, # or * and U+20E3 (keycap)
|
|
7
|
+
* - one of U+E0100–U+E01EF directly after an ideograph: an ideographic variation sequence
|
|
8
|
+
* - U+180B–U+180D or U+180F directly after a Mongolian letter
|
|
9
|
+
* - the tags of the England, Scotland and Wales flags after U+1F3F4
|
|
10
|
+
* Nothing else is benign: not U+200B, U+2060, U+FEFF, bidi controls or other tags.
|
|
11
|
+
*
|
|
12
|
+
* `counts`, `total` and `truncated` are exactly those of inspectTextDetailed;
|
|
13
|
+
* `benignCounts` counts every benign character, listed or not. Two passes over the
|
|
14
|
+
* string, no copy; memory is flat with `maxFindings` set.
|
|
15
|
+
*
|
|
16
|
+
* @param {string} text
|
|
17
|
+
* @param {InspectDetailedOptions} [options]
|
|
18
|
+
* @returns {ContextReport}
|
|
19
|
+
*/
|
|
20
|
+
export function inspectTextInContext(text: string, options?: InspectDetailedOptions): ContextReport;
|
|
21
|
+
export type TrashFinding = import("./inspectText.js").TrashFinding;
|
|
22
|
+
export type TrashCounts = import("./inspectText.js").TrashCounts;
|
|
23
|
+
export type InspectDetailedOptions = import("./inspectText.js").InspectDetailedOptions;
|
|
24
|
+
/**
|
|
25
|
+
* A finding with `benign`: true when the character is part of a legitimate sequence.
|
|
26
|
+
*/
|
|
27
|
+
export type ContextFinding = TrashFinding & {
|
|
28
|
+
benign: boolean;
|
|
29
|
+
};
|
|
30
|
+
export type ContextReport = {
|
|
31
|
+
/**
|
|
32
|
+
* - As in inspectTextDetailed, each with `benign`.
|
|
33
|
+
*/
|
|
34
|
+
findings: ContextFinding[];
|
|
35
|
+
/**
|
|
36
|
+
* - As in inspectTextDetailed: every occurrence, benign or not.
|
|
37
|
+
*/
|
|
38
|
+
counts: TrashCounts;
|
|
39
|
+
/**
|
|
40
|
+
* - Sum of `counts`.
|
|
41
|
+
*/
|
|
42
|
+
total: number;
|
|
43
|
+
/**
|
|
44
|
+
* - True when `total` exceeds `findings.length`.
|
|
45
|
+
*/
|
|
46
|
+
truncated: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* - Benign occurrences per category, listed or not.
|
|
49
|
+
* `counts[c] - benignCounts[c]` is the number of suspicious characters in category `c`.
|
|
50
|
+
*/
|
|
51
|
+
benignCounts: TrashCounts;
|
|
52
|
+
/**
|
|
53
|
+
* - Sum of `benignCounts`.
|
|
54
|
+
*/
|
|
55
|
+
benignTotal: number;
|
|
56
|
+
};
|
|
@@ -7,17 +7,26 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export function decodeTagText(text: string): string;
|
|
9
9
|
/**
|
|
10
|
-
* Finds runs of tag characters, variation selectors
|
|
10
|
+
* Finds runs of tag characters, variation selectors, invisible operators and reserved
|
|
11
|
+
* tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
|
|
11
12
|
* Consecutive characters of the same kind form one run. Two kinds of sequence are
|
|
12
13
|
* legitimate and listed under `benign` instead of `runs`:
|
|
13
14
|
* - the subdivision flags of England, Scotland and Wales: U+1F3F4 followed by the
|
|
14
15
|
* tags for "gbeng", "gbsct" or "gbwls" and U+E007F
|
|
15
16
|
* - a single U+FE0E or U+FE0F (text or emoji presentation) after another character
|
|
17
|
+
*
|
|
18
|
+
* `line` and `column` are counted as in inspectTextDetailed. Every run and benign
|
|
19
|
+
* span is counted; only the first `maxRuns` of each are listed. With the caps set,
|
|
20
|
+
* memory stays flat however large the input is. One pass over the string, no copy.
|
|
21
|
+
* Set both caps for untrusted input: without them, results grow with the input.
|
|
22
|
+
*
|
|
16
23
|
* @param {string} text
|
|
24
|
+
* @param {HiddenScanOptions} [options]
|
|
17
25
|
* @returns {HiddenTextScan}
|
|
18
26
|
*/
|
|
19
|
-
export function findHiddenRuns(text: string): HiddenTextScan;
|
|
20
|
-
export
|
|
27
|
+
export function findHiddenRuns(text: string, options?: HiddenScanOptions): HiddenTextScan;
|
|
28
|
+
export { VERSION } from "./version.generated.js";
|
|
29
|
+
export type HiddenRunKind = "tag" | "variation_selector" | "invisible_operator" | "reserved_tag_plane";
|
|
21
30
|
export type HiddenRun = {
|
|
22
31
|
kind: HiddenRunKind;
|
|
23
32
|
/**
|
|
@@ -33,18 +42,74 @@ export type HiddenRun = {
|
|
|
33
42
|
*/
|
|
34
43
|
count: number;
|
|
35
44
|
/**
|
|
36
|
-
* -
|
|
45
|
+
* - 1-based line of the first character (as in inspectTextDetailed)
|
|
46
|
+
*/
|
|
47
|
+
line: number;
|
|
48
|
+
/**
|
|
49
|
+
* - 1-based column of the first character, in UTF-16 units
|
|
50
|
+
*/
|
|
51
|
+
column: number;
|
|
52
|
+
/**
|
|
53
|
+
* - tag runs only: the ASCII the tags spell (see decodeTagText),
|
|
54
|
+
* at most `maxDecodedLength` UTF-16 units
|
|
37
55
|
*/
|
|
38
56
|
decoded?: string;
|
|
57
|
+
/**
|
|
58
|
+
* - tag runs only: true when `decoded` stops short of the run
|
|
59
|
+
*/
|
|
60
|
+
decodedTruncated?: boolean;
|
|
61
|
+
};
|
|
62
|
+
export type HiddenRunCounts = {
|
|
63
|
+
tag: number;
|
|
64
|
+
variation_selector: number;
|
|
65
|
+
invisible_operator: number;
|
|
66
|
+
reserved_tag_plane: number;
|
|
39
67
|
};
|
|
40
68
|
export type HiddenTextScan = {
|
|
41
69
|
/**
|
|
42
|
-
* - runs that may hide text, in order
|
|
70
|
+
* - runs that may hide text, in order, at most `maxRuns` long
|
|
43
71
|
*/
|
|
44
72
|
runs: HiddenRun[];
|
|
45
73
|
/**
|
|
46
74
|
* - [offset, end) spans of legitimate
|
|
47
|
-
* sequences (subdivision flags, a single emoji or text presentation selector)
|
|
75
|
+
* sequences (subdivision flags, a single emoji or text presentation selector), at most `maxRuns` long
|
|
48
76
|
*/
|
|
49
77
|
benign: Array<[number, number]>;
|
|
78
|
+
/**
|
|
79
|
+
* - every run per kind, listed or not
|
|
80
|
+
*/
|
|
81
|
+
counts: HiddenRunCounts;
|
|
82
|
+
/**
|
|
83
|
+
* - code points in those runs per kind, listed or not
|
|
84
|
+
*/
|
|
85
|
+
codePointCounts: HiddenRunCounts;
|
|
86
|
+
/**
|
|
87
|
+
* - code points U+E0100–U+E01EF in those runs, which
|
|
88
|
+
* (unlike the emoji selectors U+FE00–U+FE0F) can carry data; included in codePointCounts.variation_selector
|
|
89
|
+
*/
|
|
90
|
+
supplementarySelectorCount: number;
|
|
91
|
+
/**
|
|
92
|
+
* - sum of `counts`
|
|
93
|
+
*/
|
|
94
|
+
total: number;
|
|
95
|
+
/**
|
|
96
|
+
* - every benign span, listed or not
|
|
97
|
+
*/
|
|
98
|
+
benignTotal: number;
|
|
99
|
+
/**
|
|
100
|
+
* - true when runs or benign spans were left out of the lists
|
|
101
|
+
*/
|
|
102
|
+
truncated: boolean;
|
|
103
|
+
};
|
|
104
|
+
export type HiddenScanOptions = {
|
|
105
|
+
/**
|
|
106
|
+
* - cap on listed runs, and separately on listed benign spans
|
|
107
|
+
* (non-negative integer or Infinity). Counts stay exact.
|
|
108
|
+
*/
|
|
109
|
+
maxRuns?: number;
|
|
110
|
+
/**
|
|
111
|
+
* - cap on each run's `decoded` text, in UTF-16 units.
|
|
112
|
+
* For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
|
|
113
|
+
*/
|
|
114
|
+
maxDecodedLength?: number;
|
|
50
115
|
};
|
package/dist/types/index.d.ts
CHANGED
|
@@ -12,5 +12,11 @@ import { inspectTextDetailed } from './inspectText.js';
|
|
|
12
12
|
import { getNewlineStats } from './inspectText.js';
|
|
13
13
|
import { getRecommendedSanctifierOptions } from './inspectText.js';
|
|
14
14
|
import { summonSanctifier } from './sanctifyText.js';
|
|
15
|
+
import { sanctifyStrict } from './sanctifyText.js';
|
|
16
|
+
import { sanctifyLoose } from './sanctifyText.js';
|
|
17
|
+
import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
|
|
18
|
+
import { sanctifyKeyboardOnly } from './sanctifyText.js';
|
|
19
|
+
import { sanctifyPasted } from './sanctifyText.js';
|
|
15
20
|
import { revealInvisible } from './reveal.js';
|
|
16
|
-
|
|
21
|
+
import { VERSION } from './version.generated.js';
|
|
22
|
+
export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
|
|
@@ -1,103 +1,51 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* @
|
|
8
|
-
* @
|
|
9
|
-
* @property {boolean} [collapseNewLines]
|
|
10
|
-
* @property {boolean} [preserveParagraphs]
|
|
11
|
-
* @property {boolean} [collapseSpaces]
|
|
12
|
-
* @property {boolean} [finalTrim]
|
|
13
|
-
* @property {number} [maxLength]
|
|
14
|
-
* @property {'throw'|'truncate'|'noop'} [onMaxLength]
|
|
15
|
-
* @property {boolean} [normalizeSpaces] - Replace look-alike spaces with an ordinary space (runs first).
|
|
16
|
-
* @property {boolean} [purgeZeroWidthSpaces] - Remove U+200B, U+2060 and U+FEFF only, keeping joiners and selectors.
|
|
2
|
+
* Strict sanitizer:
|
|
3
|
+
* - Purge emojis
|
|
4
|
+
* - Collapse all newlines
|
|
5
|
+
* - Collapse spaces
|
|
6
|
+
* - Nuke control characters
|
|
7
|
+
* @param {string} text
|
|
8
|
+
* @returns {string}
|
|
17
9
|
*/
|
|
10
|
+
export function sanctifyStrict(text: string): string;
|
|
18
11
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* @param {
|
|
24
|
-
* @
|
|
25
|
-
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
26
|
-
* @param {boolean} [defaultOptions.nukeControls]
|
|
27
|
-
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
28
|
-
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
29
|
-
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
30
|
-
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
31
|
-
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
32
|
-
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
33
|
-
* @param {boolean} [defaultOptions.finalTrim]
|
|
34
|
-
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
35
|
-
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
36
|
-
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
37
|
-
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
38
|
-
* @returns {(text: string) => string}
|
|
12
|
+
* Loose sanitizer:
|
|
13
|
+
* - Collapse spaces
|
|
14
|
+
* - Preserve paragraphs
|
|
15
|
+
* - Normalize newlines
|
|
16
|
+
* @param {string} text
|
|
17
|
+
* @returns {string}
|
|
39
18
|
*/
|
|
40
|
-
export function
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
* - Normalize newlines
|
|
71
|
-
*/
|
|
72
|
-
/** @param {string} text @returns {string} */
|
|
73
|
-
function loose(text: string): string;
|
|
74
|
-
/**
|
|
75
|
-
* Keyboard-only (with emojis):
|
|
76
|
-
* - Keeps emojis and printable ASCII
|
|
77
|
-
* - Strips non-standard characters
|
|
78
|
-
* - Normalizes typographic trash
|
|
79
|
-
*/
|
|
80
|
-
/** @param {string} text @returns {string} */
|
|
81
|
-
function keyboardOnlyEmoji(text: string): string;
|
|
82
|
-
/**
|
|
83
|
-
* Keyboard-only (strict):
|
|
84
|
-
* - Removes emojis
|
|
85
|
-
* - Collapses all whitespace
|
|
86
|
-
* - Restricts to printable ASCII only
|
|
87
|
-
*/
|
|
88
|
-
/** @param {string} text @returns {string} */
|
|
89
|
-
function keyboardOnly(text: string): string;
|
|
90
|
-
/**
|
|
91
|
-
* Pasted text and model output:
|
|
92
|
-
* - Look-alike spaces (NBSP, narrow NBSP, typographic spaces) become ordinary spaces,
|
|
93
|
-
* so words never run together
|
|
94
|
-
* - Zero-width spaces, word joiners and BOMs are removed
|
|
95
|
-
* - Newlines are normalized to `\n`
|
|
96
|
-
* Keeps emoji joiners, variation selectors, bidi marks, indentation and line breaks.
|
|
97
|
-
*/
|
|
98
|
-
/** @param {string} text @returns {string} */
|
|
99
|
-
function pasted(text: string): string;
|
|
100
|
-
}
|
|
19
|
+
export function sanctifyLoose(text: string): string;
|
|
20
|
+
/**
|
|
21
|
+
* Keyboard-only (with emojis):
|
|
22
|
+
* - Keeps emojis and printable ASCII
|
|
23
|
+
* - Strips non-standard characters
|
|
24
|
+
* - Normalizes typographic trash
|
|
25
|
+
* @param {string} text
|
|
26
|
+
* @returns {string}
|
|
27
|
+
*/
|
|
28
|
+
export function sanctifyKeyboardOnlyEmoji(text: string): string;
|
|
29
|
+
/**
|
|
30
|
+
* Keyboard-only (strict):
|
|
31
|
+
* - Removes emojis
|
|
32
|
+
* - Collapses all whitespace
|
|
33
|
+
* - Restricts to printable ASCII only
|
|
34
|
+
* @param {string} text
|
|
35
|
+
* @returns {string}
|
|
36
|
+
*/
|
|
37
|
+
export function sanctifyKeyboardOnly(text: string): string;
|
|
38
|
+
/**
|
|
39
|
+
* Pasted text and model output:
|
|
40
|
+
* - Look-alike spaces (NBSP, narrow NBSP, typographic spaces) become ordinary spaces,
|
|
41
|
+
* so words never run together
|
|
42
|
+
* - Zero-width spaces, word joiners and BOMs are removed
|
|
43
|
+
* - Newlines are normalized to `\n`
|
|
44
|
+
* Keeps emoji joiners, variation selectors, bidi marks, indentation and line breaks.
|
|
45
|
+
* @param {string} text
|
|
46
|
+
* @returns {string}
|
|
47
|
+
*/
|
|
48
|
+
export function sanctifyPasted(text: string): string;
|
|
101
49
|
/**
|
|
102
50
|
* Text Sanctifier
|
|
103
51
|
*
|
|
@@ -131,11 +79,45 @@ export namespace summonSanctifier {
|
|
|
131
79
|
*/
|
|
132
80
|
export function sanctifyText(text: string, purgeInvisibleChars?: boolean, purgeEmojis?: boolean, nukeControls?: boolean, keyboardOnlyFilter?: boolean, normalizeNewlines?: boolean, trimSpacesAroundNewlines?: boolean, collapseNewLines?: boolean, preserveParagraphs?: boolean, collapseSpaces?: boolean, finalTrim?: boolean, maxLength?: number, onMaxLength?: "throw" | "truncate" | "noop", normalizeSpaces?: boolean, purgeZeroWidthSpaces?: boolean): string;
|
|
133
81
|
/**
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
82
|
+
* Summons a customized sanctifier function with pre-bound booleans.
|
|
83
|
+
*
|
|
84
|
+
* Accepts full flag names and returns a text-cleaning function.
|
|
85
|
+
*
|
|
86
|
+
* @param {Object} [defaultOptions={}]
|
|
87
|
+
* @param {boolean} [defaultOptions.purgeInvisibleChars]
|
|
88
|
+
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
89
|
+
* @param {boolean} [defaultOptions.nukeControls]
|
|
90
|
+
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
91
|
+
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
92
|
+
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
93
|
+
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
94
|
+
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
95
|
+
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
96
|
+
* @param {boolean} [defaultOptions.finalTrim]
|
|
97
|
+
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
98
|
+
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
99
|
+
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
100
|
+
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
101
|
+
* @returns {(text: string) => string}
|
|
102
|
+
*
|
|
103
|
+
* The presets are attached as properties (`summonSanctifier.strict`, `.loose`,
|
|
104
|
+
* `.keyboardOnlyEmoji`, `.keyboardOnly`, `.pasted`) and are also standalone exports.
|
|
105
|
+
* Built in a call marked pure, so a bundle that imports only a preset leaves it out.
|
|
106
|
+
* @type {typeof createSanctifier & {
|
|
107
|
+
* strict: typeof sanctifyStrict,
|
|
108
|
+
* loose: typeof sanctifyLoose,
|
|
109
|
+
* keyboardOnlyEmoji: typeof sanctifyKeyboardOnlyEmoji,
|
|
110
|
+
* keyboardOnly: typeof sanctifyKeyboardOnly,
|
|
111
|
+
* pasted: typeof sanctifyPasted,
|
|
112
|
+
* }}
|
|
137
113
|
*/
|
|
138
|
-
export
|
|
114
|
+
export const summonSanctifier: typeof createSanctifier & {
|
|
115
|
+
strict: typeof sanctifyStrict;
|
|
116
|
+
loose: typeof sanctifyLoose;
|
|
117
|
+
keyboardOnlyEmoji: typeof sanctifyKeyboardOnlyEmoji;
|
|
118
|
+
keyboardOnly: typeof sanctifyKeyboardOnly;
|
|
119
|
+
pasted: typeof sanctifyPasted;
|
|
120
|
+
};
|
|
139
121
|
/**
|
|
140
122
|
* Look-alike spaces: the `nonbreaking_space` category except U+3000, the ordinary
|
|
141
123
|
* space of CJK text. A test checks this against the category table.
|
|
@@ -143,15 +125,6 @@ export function normalizeTypographicJank(text: string): string;
|
|
|
143
125
|
export const SPACE_LOOKALIKES_REGEX: RegExp;
|
|
144
126
|
/** Zero-width space, word joiner and BOM: invisible and never needed in plain text. */
|
|
145
127
|
export const ZERO_WIDTH_SPACES_REGEX: RegExp;
|
|
146
|
-
/**
|
|
147
|
-
* Matches any character that is NOT:
|
|
148
|
-
* - Printable ASCII (U+0020–U+007E)
|
|
149
|
-
* - Newline characters: \n (LF), \r (CR)
|
|
150
|
-
* This allows for \n, \r, and \r\n line endings.
|
|
151
|
-
*/
|
|
152
|
-
export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
|
|
153
|
-
/** @type {RegExp} */
|
|
154
|
-
export let EMOJI_REGEX: RegExp;
|
|
155
128
|
export type SanctifyOptions = {
|
|
156
129
|
purgeInvisibleChars?: boolean;
|
|
157
130
|
purgeEmojis?: boolean;
|
|
@@ -176,4 +149,62 @@ export type SanctifyOptions = {
|
|
|
176
149
|
};
|
|
177
150
|
import { CONTROL_CHARS_REGEX } from './categories.generated.js';
|
|
178
151
|
import { INVISIBLE_TRASH_REGEX } from './categories.generated.js';
|
|
179
|
-
|
|
152
|
+
import { ASCII_KEYBOARD_SAFE_REGEX } from './textPatterns.js';
|
|
153
|
+
import { EMOJI_REGEX } from './textPatterns.js';
|
|
154
|
+
import { normalizeTypographicJank } from './textPatterns.js';
|
|
155
|
+
/**
|
|
156
|
+
* @typedef {Object} SanctifyOptions
|
|
157
|
+
* @property {boolean} [purgeInvisibleChars]
|
|
158
|
+
* @property {boolean} [purgeEmojis]
|
|
159
|
+
* @property {boolean} [nukeControls]
|
|
160
|
+
* @property {boolean} [keyboardOnlyFilter]
|
|
161
|
+
* @property {boolean} [normalizeNewlines]
|
|
162
|
+
* @property {boolean} [trimSpacesAroundNewlines]
|
|
163
|
+
* @property {boolean} [collapseNewLines]
|
|
164
|
+
* @property {boolean} [preserveParagraphs]
|
|
165
|
+
* @property {boolean} [collapseSpaces]
|
|
166
|
+
* @property {boolean} [finalTrim]
|
|
167
|
+
* @property {number} [maxLength]
|
|
168
|
+
* @property {'throw'|'truncate'|'noop'} [onMaxLength]
|
|
169
|
+
* @property {boolean} [normalizeSpaces] - Replace look-alike spaces with an ordinary space (runs first).
|
|
170
|
+
* @property {boolean} [purgeZeroWidthSpaces] - Remove U+200B, U+2060 and U+FEFF only, keeping joiners and selectors.
|
|
171
|
+
*/
|
|
172
|
+
/**
|
|
173
|
+
* Summons a customized sanctifier function with pre-bound booleans.
|
|
174
|
+
*
|
|
175
|
+
* Accepts full flag names and returns a text-cleaning function.
|
|
176
|
+
*
|
|
177
|
+
* @param {Object} [defaultOptions={}]
|
|
178
|
+
* @param {boolean} [defaultOptions.purgeInvisibleChars]
|
|
179
|
+
* @param {boolean} [defaultOptions.purgeEmojis]
|
|
180
|
+
* @param {boolean} [defaultOptions.nukeControls]
|
|
181
|
+
* @param {boolean} [defaultOptions.keyboardOnlyFilter]
|
|
182
|
+
* @param {boolean} [defaultOptions.normalizeNewlines]
|
|
183
|
+
* @param {boolean} [defaultOptions.trimSpacesAroundNewlines]
|
|
184
|
+
* @param {boolean} [defaultOptions.collapseNewLines]
|
|
185
|
+
* @param {boolean} [defaultOptions.preserveParagraphs]
|
|
186
|
+
* @param {boolean} [defaultOptions.collapseSpaces]
|
|
187
|
+
* @param {boolean} [defaultOptions.finalTrim]
|
|
188
|
+
* @param {number} [defaultOptions.maxLength=Infinity] - Hard input length cap (UTF-16 code units).
|
|
189
|
+
* @param {'throw' | 'truncate' | 'noop'} [defaultOptions.onMaxLength='throw'] - Behavior when input exceeds maxLength.
|
|
190
|
+
* @param {boolean} [defaultOptions.normalizeSpaces]
|
|
191
|
+
* @param {boolean} [defaultOptions.purgeZeroWidthSpaces]
|
|
192
|
+
* @returns {(text: string) => string}
|
|
193
|
+
*/
|
|
194
|
+
declare function createSanctifier(defaultOptions?: {
|
|
195
|
+
purgeInvisibleChars?: boolean;
|
|
196
|
+
purgeEmojis?: boolean;
|
|
197
|
+
nukeControls?: boolean;
|
|
198
|
+
keyboardOnlyFilter?: boolean;
|
|
199
|
+
normalizeNewlines?: boolean;
|
|
200
|
+
trimSpacesAroundNewlines?: boolean;
|
|
201
|
+
collapseNewLines?: boolean;
|
|
202
|
+
preserveParagraphs?: boolean;
|
|
203
|
+
collapseSpaces?: boolean;
|
|
204
|
+
finalTrim?: boolean;
|
|
205
|
+
maxLength?: number;
|
|
206
|
+
onMaxLength?: "throw" | "truncate" | "noop";
|
|
207
|
+
normalizeSpaces?: boolean;
|
|
208
|
+
purgeZeroWidthSpaces?: boolean;
|
|
209
|
+
}): (text: string) => string;
|
|
210
|
+
export { CONTROL_CHARS_REGEX, INVISIBLE_TRASH_REGEX, ASCII_KEYBOARD_SAFE_REGEX, EMOJI_REGEX, normalizeTypographicJank };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Normalizes typographic Unicode punctuation into ASCII equivalents.
|
|
3
|
+
* @param {string} text
|
|
4
|
+
* @returns {string}
|
|
5
|
+
*/
|
|
6
|
+
export function normalizeTypographicJank(text: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* Matches any character that is NOT:
|
|
9
|
+
* - Printable ASCII (U+0020–U+007E)
|
|
10
|
+
* - Newline characters: \n (LF), \r (CR)
|
|
11
|
+
* This allows for \n, \r, and \r\n line endings.
|
|
12
|
+
*/
|
|
13
|
+
export const ASCII_KEYBOARD_SAFE_REGEX: RegExp;
|
|
14
|
+
/** @type {RegExp} */
|
|
15
|
+
export const EMOJI_REGEX: RegExp;
|