text-sanctifier 1.3.9 → 1.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -84
- package/bin/text-sanctifier.js +214 -69
- package/dist/text-sanctifier.min.js +5 -5
- package/dist/types/hiddenScan.d.ts +30 -0
- package/dist/types/hiddenText.d.ts +40 -0
- package/dist/types/inspectText.d.ts +4 -1
- package/dist/types/sanctifyText.d.ts +1 -1
- package/dist/types/version.generated.d.ts +1 -1
- package/package.json +8 -6
- package/src/hiddenScan.js +171 -0
- package/src/hiddenText.js +4 -151
- package/src/inspectText.js +7 -3
- package/src/sanctifyText.js +17 -11
- package/src/version.generated.js +1 -1
- package/src/index.d.ts +0 -22
|
@@ -68,7 +68,7 @@ export function sanctifyPasted(text: string): string;
|
|
|
68
68
|
* @param {number} [maxLength=Infinity] - Hard input length cap (UTF-16 code units via `text.length`).
|
|
69
69
|
* @param {'throw'|'truncate'|'noop'} [onMaxLength='throw'] - Behavior when `text.length` exceeds `maxLength`:
|
|
70
70
|
* - `'throw'`: throw a `RangeError`
|
|
71
|
-
* - `'truncate'`: slice to `maxLength` before processing
|
|
71
|
+
* - `'truncate'`: slice to at most `maxLength` before processing, never through a surrogate pair
|
|
72
72
|
* - `'noop'`: return the original input unchanged (skips processing)
|
|
73
73
|
* @param {boolean} [normalizeSpaces=false] - Replace look-alike spaces with an ordinary space. Runs first,
|
|
74
74
|
* so `purgeInvisibleChars` no longer joins the words around them.
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** The installed version of text-sanctifier, as in its package.json. */
|
|
2
|
-
export const VERSION: "1.3.
|
|
2
|
+
export const VERSION: "1.3.11";
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "text-sanctifier",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.11",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "Find and remove invisible Unicode in web input, model-bound text and repositories.",
|
|
6
6
|
"engines": {
|
|
7
7
|
"node": ">=18"
|
|
8
8
|
},
|
|
@@ -48,15 +48,17 @@
|
|
|
48
48
|
"build": "node scripts/build.js",
|
|
49
49
|
"build:types": "tsc -p tsconfig.types.json",
|
|
50
50
|
"test:types": "tsc -p tests/types/tsconfig.json",
|
|
51
|
-
"test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js && node tests/release.test.js",
|
|
51
|
+
"test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js && node tests/release.test.js && node tests/names.test.js && node tests/docs.test.js && node tests/repo.test.js",
|
|
52
52
|
"test-min": "node tests/sanctifyText.test.min.js",
|
|
53
53
|
"check:self": "node bin/text-sanctifier.js check .",
|
|
54
54
|
"check:names": "node scripts/check-names.js",
|
|
55
|
-
"
|
|
55
|
+
"check:repo": "node scripts/check-repo.js",
|
|
56
|
+
"ci": "npm run gen:check && npm run check:names && npm run check:repo && npm run check:self && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
|
|
56
57
|
"size": "node scripts/check-size.js",
|
|
57
58
|
"bench": "node --expose-gc scripts/bench.js",
|
|
58
59
|
"release:prep": "node scripts/release.js prep",
|
|
59
60
|
"release:verify": "node scripts/release.js verify",
|
|
61
|
+
"prepublishOnly": "node scripts/release.js check-publish",
|
|
60
62
|
"hooks:install": "git config core.hooksPath scripts/hooks"
|
|
61
63
|
},
|
|
62
64
|
"keywords": [
|
|
@@ -83,7 +85,7 @@
|
|
|
83
85
|
"test": "tests"
|
|
84
86
|
},
|
|
85
87
|
"devDependencies": {
|
|
86
|
-
"esbuild": "
|
|
87
|
-
"typescript": "
|
|
88
|
+
"esbuild": "0.25.12",
|
|
89
|
+
"typescript": "5.9.3"
|
|
88
90
|
}
|
|
89
91
|
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// Internal hidden-span scanner; not a package entry point.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @typedef {import('./hiddenText.js').HiddenRunKind} HiddenRunKind
|
|
5
|
+
* @typedef {import('./hiddenText.js').HiddenRun} HiddenRun
|
|
6
|
+
* @typedef {import('./hiddenText.js').HiddenRunCounts} HiddenRunCounts
|
|
7
|
+
* @typedef {import('./hiddenText.js').HiddenTextScan} HiddenTextScan
|
|
8
|
+
* @typedef {import('./hiddenText.js').HiddenScanOptions} HiddenScanOptions
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const TAG_FIRST = 0xE0000;
|
|
12
|
+
const TAG_LAST = 0xE007F;
|
|
13
|
+
const CANCEL_TAG = 0xE007F;
|
|
14
|
+
const BLACK_FLAG = 0x1F3F4;
|
|
15
|
+
|
|
16
|
+
// RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
|
|
17
|
+
const SUBDIVISION_FLAGS = new Set(['gbeng', 'gbsct', 'gbwls']);
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {number} cp
|
|
21
|
+
* @returns {HiddenRunKind | null}
|
|
22
|
+
*/
|
|
23
|
+
function kindOf(cp) {
|
|
24
|
+
if (cp >= TAG_FIRST && cp <= TAG_LAST) return 'tag';
|
|
25
|
+
if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
|
|
26
|
+
(cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
|
|
27
|
+
if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
|
|
28
|
+
// Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
|
|
29
|
+
if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** @param {number} cp */
|
|
34
|
+
const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Decodes text[start, end) as decodeTagText does, stopping before the output would
|
|
38
|
+
* pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
|
|
39
|
+
* @param {string} text
|
|
40
|
+
* @param {number} start
|
|
41
|
+
* @param {number} end
|
|
42
|
+
* @param {number} limit
|
|
43
|
+
* @returns {{ decoded: string, truncated: boolean }}
|
|
44
|
+
*/
|
|
45
|
+
export function decodeRange(text, start, end, limit) {
|
|
46
|
+
let out = '';
|
|
47
|
+
let i = start;
|
|
48
|
+
while (i < end) {
|
|
49
|
+
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
50
|
+
const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
|
|
51
|
+
if (out.length + piece.length > limit) return { decoded: out, truncated: true };
|
|
52
|
+
out += piece;
|
|
53
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
54
|
+
}
|
|
55
|
+
return { decoded: out, truncated: false };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Tested at an index of the text itself, so the check allocates nothing.
|
|
59
|
+
const PICTOGRAPHIC_AT = /\p{Extended_Pictographic}/yu;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* True when a presentation selector at `index` follows an emoji-capable character:
|
|
63
|
+
* an Extended_Pictographic code point, or a keycap base (#, * or a digit) with U+20E3 after.
|
|
64
|
+
* @param {string} text
|
|
65
|
+
* @param {number} index
|
|
66
|
+
* @returns {boolean}
|
|
67
|
+
*/
|
|
68
|
+
function isPresentationBase(text, index) {
|
|
69
|
+
if (index === 0) return false;
|
|
70
|
+
const low = text.charCodeAt(index - 1);
|
|
71
|
+
const p = low >= 0xDC00 && low <= 0xDFFF && index >= 2 &&
|
|
72
|
+
(text.charCodeAt(index - 2) & 0xFC00) === 0xD800 ? index - 2 : index - 1;
|
|
73
|
+
const before = text.charCodeAt(p);
|
|
74
|
+
if ((before === 0x23 || before === 0x2A || (before >= 0x30 && before <= 0x39)) && text.charCodeAt(index + 1) === 0x20E3) return true;
|
|
75
|
+
PICTOGRAPHIC_AT.lastIndex = p;
|
|
76
|
+
return PICTOGRAPHIC_AT.test(text);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* A cap option: a non-negative integer or Infinity; anything else means no cap.
|
|
81
|
+
* @param {unknown} value
|
|
82
|
+
* @returns {number}
|
|
83
|
+
*/
|
|
84
|
+
function cap(value) {
|
|
85
|
+
return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
|
|
86
|
+
? value
|
|
87
|
+
: Infinity;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* @param {string} text
|
|
92
|
+
* @param {HiddenScanOptions} [options]
|
|
93
|
+
* @param {(span: {offset: number, end: number, benign: boolean, kind: HiddenRunKind}) => void} [onSpan] - internal streaming visitor
|
|
94
|
+
* @returns {HiddenTextScan}
|
|
95
|
+
*/
|
|
96
|
+
export function scanHiddenSpans(text, options = {}, onSpan) {
|
|
97
|
+
if (typeof text !== 'string') throw new TypeError('findHiddenRuns expects a string input.');
|
|
98
|
+
const maxRuns = cap(options && options.maxRuns);
|
|
99
|
+
const maxDecodedLength = cap(options && options.maxDecodedLength);
|
|
100
|
+
|
|
101
|
+
/** @type {HiddenRun[]} */
|
|
102
|
+
const runs = [];
|
|
103
|
+
/** @type {Array<[number, number]>} */
|
|
104
|
+
const benign = [];
|
|
105
|
+
/** @type {HiddenRunCounts} */
|
|
106
|
+
const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
|
|
107
|
+
/** @type {HiddenRunCounts} */
|
|
108
|
+
const codePointCounts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
|
|
109
|
+
let supplementarySelectorCount = 0;
|
|
110
|
+
let total = 0;
|
|
111
|
+
let benignTotal = 0;
|
|
112
|
+
let line = 1;
|
|
113
|
+
let lineStart = 0;
|
|
114
|
+
|
|
115
|
+
let i = 0;
|
|
116
|
+
while (i < text.length) {
|
|
117
|
+
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
118
|
+
const kind = kindOf(cp);
|
|
119
|
+
if (kind === null) {
|
|
120
|
+
if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
|
|
121
|
+
line++;
|
|
122
|
+
lineStart = i + 1;
|
|
123
|
+
}
|
|
124
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const offset = i;
|
|
129
|
+
let count = 0;
|
|
130
|
+
let supplementary = 0;
|
|
131
|
+
let last = cp;
|
|
132
|
+
while (i < text.length) {
|
|
133
|
+
const c = /** @type {number} */ (text.codePointAt(i));
|
|
134
|
+
if (kindOf(c) !== kind) break;
|
|
135
|
+
last = c;
|
|
136
|
+
count++;
|
|
137
|
+
if (c >= 0xE0100 && c <= 0xE01EF) supplementary++;
|
|
138
|
+
i += c > 0xFFFF ? 2 : 1;
|
|
139
|
+
}
|
|
140
|
+
const end = i;
|
|
141
|
+
|
|
142
|
+
// A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
|
|
143
|
+
const isBenign = kind === 'tag'
|
|
144
|
+
? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
|
|
145
|
+
SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
|
|
146
|
+
: kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
|
|
147
|
+
isPresentationBase(text, offset);
|
|
148
|
+
if (onSpan) onSpan({ offset, end, benign: isBenign, kind });
|
|
149
|
+
if (isBenign) {
|
|
150
|
+
benignTotal++;
|
|
151
|
+
if (benign.length < maxRuns) benign.push([offset, end]);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
counts[kind]++;
|
|
156
|
+
codePointCounts[kind] += count;
|
|
157
|
+
supplementarySelectorCount += supplementary;
|
|
158
|
+
total++;
|
|
159
|
+
if (runs.length < maxRuns) {
|
|
160
|
+
/** @type {HiddenRun} */
|
|
161
|
+
const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
|
|
162
|
+
if (kind === 'tag') {
|
|
163
|
+
const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
|
|
164
|
+
run.decoded = decoded;
|
|
165
|
+
run.decodedTruncated = truncated;
|
|
166
|
+
}
|
|
167
|
+
runs.push(run);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return { runs, benign, counts, codePointCounts, supplementarySelectorCount, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
|
|
171
|
+
}
|
package/src/hiddenText.js
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
// Plain JavaScript with no Node APIs, so it also runs in a browser. src/index.js does
|
|
10
10
|
// not import this file, so it stays out of the browser bundle and its size budget.
|
|
11
11
|
|
|
12
|
+
import { scanHiddenSpans, decodeRange } from './hiddenScan.js';
|
|
13
|
+
|
|
12
14
|
export { VERSION } from './version.generated.js';
|
|
13
15
|
|
|
14
16
|
/** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
|
|
@@ -56,31 +58,6 @@ export { VERSION } from './version.generated.js';
|
|
|
56
58
|
* For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
|
|
57
59
|
*/
|
|
58
60
|
|
|
59
|
-
const TAG_FIRST = 0xE0000;
|
|
60
|
-
const TAG_LAST = 0xE007F;
|
|
61
|
-
const CANCEL_TAG = 0xE007F;
|
|
62
|
-
const BLACK_FLAG = 0x1F3F4;
|
|
63
|
-
|
|
64
|
-
// RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
|
|
65
|
-
const SUBDIVISION_FLAGS = new Set(['gbeng', 'gbsct', 'gbwls']);
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* @param {number} cp
|
|
69
|
-
* @returns {HiddenRunKind | null}
|
|
70
|
-
*/
|
|
71
|
-
function kindOf(cp) {
|
|
72
|
-
if (cp >= TAG_FIRST && cp <= TAG_LAST) return 'tag';
|
|
73
|
-
if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
|
|
74
|
-
(cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
|
|
75
|
-
if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
|
|
76
|
-
// Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
|
|
77
|
-
if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
|
|
78
|
-
return null;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** @param {number} cp */
|
|
82
|
-
const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
|
|
83
|
-
|
|
84
61
|
/**
|
|
85
62
|
* Decodes tag characters to the ASCII they shadow: U+E0020–U+E007E become
|
|
86
63
|
* U+0020–U+007E. Anything else (U+E0001 LANGUAGE TAG, U+E007F CANCEL TAG, unassigned
|
|
@@ -89,63 +66,10 @@ const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
|
|
|
89
66
|
* @returns {string}
|
|
90
67
|
*/
|
|
91
68
|
export function decodeTagText(text) {
|
|
69
|
+
if (typeof text !== 'string') throw new TypeError('decodeTagText expects a string input.');
|
|
92
70
|
return decodeRange(text, 0, text.length, Infinity).decoded;
|
|
93
71
|
}
|
|
94
72
|
|
|
95
|
-
/**
|
|
96
|
-
* Decodes text[start, end) as decodeTagText does, stopping before the output would
|
|
97
|
-
* pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
|
|
98
|
-
* @param {string} text
|
|
99
|
-
* @param {number} start
|
|
100
|
-
* @param {number} end
|
|
101
|
-
* @param {number} limit
|
|
102
|
-
* @returns {{ decoded: string, truncated: boolean }}
|
|
103
|
-
*/
|
|
104
|
-
function decodeRange(text, start, end, limit) {
|
|
105
|
-
let out = '';
|
|
106
|
-
let i = start;
|
|
107
|
-
while (i < end) {
|
|
108
|
-
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
109
|
-
const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
|
|
110
|
-
if (out.length + piece.length > limit) return { decoded: out, truncated: true };
|
|
111
|
-
out += piece;
|
|
112
|
-
i += cp > 0xFFFF ? 2 : 1;
|
|
113
|
-
}
|
|
114
|
-
return { decoded: out, truncated: false };
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// Tested at an index of the text itself, so the check allocates nothing.
|
|
118
|
-
const PICTOGRAPHIC_AT = /\p{Extended_Pictographic}/yu;
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* True when a presentation selector at `index` follows an emoji-capable character:
|
|
122
|
-
* an Extended_Pictographic code point, or a keycap base (#, * or a digit) with U+20E3 after.
|
|
123
|
-
* @param {string} text
|
|
124
|
-
* @param {number} index
|
|
125
|
-
* @returns {boolean}
|
|
126
|
-
*/
|
|
127
|
-
function isPresentationBase(text, index) {
|
|
128
|
-
if (index === 0) return false;
|
|
129
|
-
const low = text.charCodeAt(index - 1);
|
|
130
|
-
const p = low >= 0xDC00 && low <= 0xDFFF && index >= 2 &&
|
|
131
|
-
(text.charCodeAt(index - 2) & 0xFC00) === 0xD800 ? index - 2 : index - 1;
|
|
132
|
-
const before = text.charCodeAt(p);
|
|
133
|
-
if ((before === 0x23 || before === 0x2A || (before >= 0x30 && before <= 0x39)) && text.charCodeAt(index + 1) === 0x20E3) return true;
|
|
134
|
-
PICTOGRAPHIC_AT.lastIndex = p;
|
|
135
|
-
return PICTOGRAPHIC_AT.test(text);
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/**
|
|
139
|
-
* A cap option: a non-negative integer or Infinity; anything else means no cap.
|
|
140
|
-
* @param {unknown} value
|
|
141
|
-
* @returns {number}
|
|
142
|
-
*/
|
|
143
|
-
function cap(value) {
|
|
144
|
-
return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
|
|
145
|
-
? value
|
|
146
|
-
: Infinity;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
73
|
/**
|
|
150
74
|
* Finds runs of tag characters, variation selectors, invisible operators and reserved
|
|
151
75
|
* tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
|
|
@@ -168,76 +92,5 @@ function cap(value) {
|
|
|
168
92
|
* @returns {HiddenTextScan}
|
|
169
93
|
*/
|
|
170
94
|
export function findHiddenRuns(text, options = {}) {
|
|
171
|
-
|
|
172
|
-
const maxDecodedLength = cap(options && options.maxDecodedLength);
|
|
173
|
-
|
|
174
|
-
/** @type {HiddenRun[]} */
|
|
175
|
-
const runs = [];
|
|
176
|
-
/** @type {Array<[number, number]>} */
|
|
177
|
-
const benign = [];
|
|
178
|
-
/** @type {HiddenRunCounts} */
|
|
179
|
-
const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
|
|
180
|
-
/** @type {HiddenRunCounts} */
|
|
181
|
-
const codePointCounts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
|
|
182
|
-
let supplementarySelectorCount = 0;
|
|
183
|
-
let total = 0;
|
|
184
|
-
let benignTotal = 0;
|
|
185
|
-
let line = 1;
|
|
186
|
-
let lineStart = 0;
|
|
187
|
-
|
|
188
|
-
let i = 0;
|
|
189
|
-
while (i < text.length) {
|
|
190
|
-
const cp = /** @type {number} */ (text.codePointAt(i));
|
|
191
|
-
const kind = kindOf(cp);
|
|
192
|
-
if (kind === null) {
|
|
193
|
-
if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
|
|
194
|
-
line++;
|
|
195
|
-
lineStart = i + 1;
|
|
196
|
-
}
|
|
197
|
-
i += cp > 0xFFFF ? 2 : 1;
|
|
198
|
-
continue;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
const offset = i;
|
|
202
|
-
let count = 0;
|
|
203
|
-
let supplementary = 0;
|
|
204
|
-
let last = cp;
|
|
205
|
-
while (i < text.length) {
|
|
206
|
-
const c = /** @type {number} */ (text.codePointAt(i));
|
|
207
|
-
if (kindOf(c) !== kind) break;
|
|
208
|
-
last = c;
|
|
209
|
-
count++;
|
|
210
|
-
if (c >= 0xE0100 && c <= 0xE01EF) supplementary++;
|
|
211
|
-
i += c > 0xFFFF ? 2 : 1;
|
|
212
|
-
}
|
|
213
|
-
const end = i;
|
|
214
|
-
|
|
215
|
-
// A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
|
|
216
|
-
const isBenign = kind === 'tag'
|
|
217
|
-
? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
|
|
218
|
-
SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
|
|
219
|
-
: kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
|
|
220
|
-
isPresentationBase(text, offset);
|
|
221
|
-
if (isBenign) {
|
|
222
|
-
benignTotal++;
|
|
223
|
-
if (benign.length < maxRuns) benign.push([offset, end]);
|
|
224
|
-
continue;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
counts[kind]++;
|
|
228
|
-
codePointCounts[kind] += count;
|
|
229
|
-
supplementarySelectorCount += supplementary;
|
|
230
|
-
total++;
|
|
231
|
-
if (runs.length < maxRuns) {
|
|
232
|
-
/** @type {HiddenRun} */
|
|
233
|
-
const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
|
|
234
|
-
if (kind === 'tag') {
|
|
235
|
-
const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
|
|
236
|
-
run.decoded = decoded;
|
|
237
|
-
run.decodedTruncated = truncated;
|
|
238
|
-
}
|
|
239
|
-
runs.push(run);
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
return { runs, benign, counts, codePointCounts, supplementarySelectorCount, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
|
|
95
|
+
return scanHiddenSpans(text, options);
|
|
243
96
|
}
|
package/src/inspectText.js
CHANGED
|
@@ -322,16 +322,20 @@ export function getNewlineStats(text) {
|
|
|
322
322
|
}
|
|
323
323
|
|
|
324
324
|
/**
|
|
325
|
-
* Creates defaultOptions for summonSanctifier based on inspectText result
|
|
325
|
+
* Creates defaultOptions for summonSanctifier based on inspectText result.
|
|
326
|
+
* Recommends only what removes trash without losing meaning: non-ASCII text and emoji are
|
|
327
|
+
* content, so `purgeEmojis` and `keyboardOnlyFilter` stay opt-in, and look-alike spaces
|
|
328
|
+
* become ordinary spaces so that words do not run together.
|
|
326
329
|
* @param {!UnicodeTrashReport} report
|
|
327
330
|
* @return {!SanctifyOptions}
|
|
328
331
|
*/
|
|
329
332
|
export function getRecommendedSanctifierOptions(report) {
|
|
330
333
|
return {
|
|
334
|
+
normalizeSpaces: report.hasInvisibleChars,
|
|
331
335
|
purgeInvisibleChars: report.hasInvisibleChars,
|
|
332
|
-
purgeEmojis:
|
|
336
|
+
purgeEmojis: false,
|
|
333
337
|
nukeControls: report.hasControlChars,
|
|
334
|
-
keyboardOnlyFilter:
|
|
338
|
+
keyboardOnlyFilter: false,
|
|
335
339
|
normalizeNewlines:
|
|
336
340
|
report.hasMixedNewlines || report.newlineStyle === 'CRLF' || report.newlineStyle === 'CR',
|
|
337
341
|
};
|
package/src/sanctifyText.js
CHANGED
|
@@ -294,7 +294,7 @@ export const summonSanctifier = /* @__PURE__ */ Object.assign(createSanctifier,
|
|
|
294
294
|
* @param {number} [maxLength=Infinity] - Hard input length cap (UTF-16 code units via `text.length`).
|
|
295
295
|
* @param {'throw'|'truncate'|'noop'} [onMaxLength='throw'] - Behavior when `text.length` exceeds `maxLength`:
|
|
296
296
|
* - `'throw'`: throw a `RangeError`
|
|
297
|
-
* - `'truncate'`: slice to `maxLength` before processing
|
|
297
|
+
* - `'truncate'`: slice to at most `maxLength` before processing, never through a surrogate pair
|
|
298
298
|
* - `'noop'`: return the original input unchanged (skips processing)
|
|
299
299
|
* @param {boolean} [normalizeSpaces=false] - Replace look-alike spaces with an ordinary space. Runs first,
|
|
300
300
|
* so `purgeInvisibleChars` no longer joins the words around them.
|
|
@@ -324,26 +324,27 @@ export function sanctifyText(
|
|
|
324
324
|
throw new TypeError('sanctifyText expects a string input.');
|
|
325
325
|
}
|
|
326
326
|
|
|
327
|
-
if (onMaxLength !== 'truncate' && onMaxLength !== 'noop' && onMaxLength !== 'throw') {
|
|
328
|
-
onMaxLength = 'throw';
|
|
329
|
-
}
|
|
330
|
-
|
|
331
327
|
if (text.length > maxLength) {
|
|
332
328
|
switch (onMaxLength) {
|
|
333
329
|
case 'truncate':
|
|
334
|
-
|
|
330
|
+
{
|
|
331
|
+
// Cut before a surrogate pair, not through it
|
|
332
|
+
const last = text.charCodeAt(maxLength - 1);
|
|
333
|
+
text = text.slice(0, last >= 0xD800 && last <= 0xDBFF ? maxLength - 1 : maxLength);
|
|
334
|
+
}
|
|
335
335
|
break;
|
|
336
336
|
case 'noop':
|
|
337
337
|
return text;
|
|
338
|
-
|
|
339
|
-
default:
|
|
338
|
+
default: // 'throw', and any invalid value
|
|
340
339
|
throw new RangeError(
|
|
341
340
|
`sanctifyText input length ${text.length} exceeds maxLength ${maxLength}.`
|
|
342
341
|
);
|
|
343
342
|
}
|
|
344
343
|
}
|
|
345
344
|
|
|
346
|
-
|
|
345
|
+
// Unpaired surrogates become U+FFFD first, so removing a character between two
|
|
346
|
+
// halves cannot join them into a new character, such as a hidden tag.
|
|
347
|
+
let cleaned = text.replace(LONE_SURROGATES_REGEX, '\uFFFD');
|
|
347
348
|
|
|
348
349
|
// Look-alike spaces become ordinary spaces, before anything can delete them
|
|
349
350
|
if (normalizeSpaces) cleaned = cleaned.replace(SPACE_LOOKALIKES_REGEX, ' ');
|
|
@@ -389,6 +390,9 @@ export function sanctifyText(
|
|
|
389
390
|
*/
|
|
390
391
|
export const SPACE_LOOKALIKES_REGEX = /[\u00A0\u2000-\u200A\u202F\u205F]/g;
|
|
391
392
|
|
|
393
|
+
/** With the `u` flag a surrogate pair is one character, so this matches only unpaired halves. */
|
|
394
|
+
const LONE_SURROGATES_REGEX = /[\uD800-\uDFFF]/gu;
|
|
395
|
+
|
|
392
396
|
/** Zero-width space, word joiner and BOM: invisible and never needed in plain text. */
|
|
393
397
|
export const ZERO_WIDTH_SPACES_REGEX = /[\u200B\u2060\uFEFF]/g;
|
|
394
398
|
|
|
@@ -497,9 +501,11 @@ function normalizeNewlineChars(text, normalized = '\n') {
|
|
|
497
501
|
* @param {string} text
|
|
498
502
|
* @returns {string}
|
|
499
503
|
*/
|
|
500
|
-
|
|
504
|
+
// The second branch consumes a run with no newline after it, so matching resumes past the
|
|
505
|
+
// run instead of retrying from each of its characters, which took quadratic time.
|
|
506
|
+
const TRIM_SPACES_AROUND_NEWLINES_REGEX = /[ \t]*(\n+)[ \t]*|([ \t]+)/g;
|
|
501
507
|
function trimSpacesAroundNewlineChars(text) {
|
|
502
|
-
return text.replace(TRIM_SPACES_AROUND_NEWLINES_REGEX, '$1');
|
|
508
|
+
return text.replace(TRIM_SPACES_AROUND_NEWLINES_REGEX, '$1$2');
|
|
503
509
|
}
|
|
504
510
|
|
|
505
511
|
|
package/src/version.generated.js
CHANGED
package/src/index.d.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { inspectText } from './inspectText.js';
|
|
2
|
-
import { inspectTextDetailed } from './inspectText.js';
|
|
3
|
-
import { getNewlineStats } from './inspectText.js';
|
|
4
|
-
import { getRecommendedSanctifierOptions } from './inspectText.js';
|
|
5
|
-
import { summonSanctifier } from './sanctifyText.js';
|
|
6
|
-
import { sanctifyStrict } from './sanctifyText.js';
|
|
7
|
-
import { sanctifyLoose } from './sanctifyText.js';
|
|
8
|
-
import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
|
|
9
|
-
import { sanctifyKeyboardOnly } from './sanctifyText.js';
|
|
10
|
-
import { sanctifyPasted } from './sanctifyText.js';
|
|
11
|
-
import { revealInvisible } from './reveal.js';
|
|
12
|
-
import { VERSION } from './version.generated.js';
|
|
13
|
-
export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
|
|
14
|
-
export type UnicodeTrashReport = import('./inspectText.js').UnicodeTrashReport;
|
|
15
|
-
export type TrashCategory = import('./inspectText.js').TrashCategory;
|
|
16
|
-
export type TrashFinding = import('./inspectText.js').TrashFinding;
|
|
17
|
-
export type TrashCounts = import('./inspectText.js').TrashCounts;
|
|
18
|
-
export type DetailedTrashReport = import('./inspectText.js').DetailedTrashReport;
|
|
19
|
-
export type InspectDetailedOptions = import('./inspectText.js').InspectDetailedOptions;
|
|
20
|
-
export type NewlineStats = import('./inspectText.js').NewlineStats;
|
|
21
|
-
export type SanctifyOptions = import('./sanctifyText.js').SanctifyOptions;
|
|
22
|
-
export type RevealOptions = import('./reveal.js').RevealOptions;
|