text-sanctifier 1.3.9 → 1.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -68,7 +68,7 @@ export function sanctifyPasted(text: string): string;
68
68
  * @param {number} [maxLength=Infinity] - Hard input length cap (UTF-16 code units via `text.length`).
69
69
  * @param {'throw'|'truncate'|'noop'} [onMaxLength='throw'] - Behavior when `text.length` exceeds `maxLength`:
70
70
  * - `'throw'`: throw a `RangeError`
71
- * - `'truncate'`: slice to `maxLength` before processing
71
+ * - `'truncate'`: slice to at most `maxLength` before processing, never through a surrogate pair
72
72
  * - `'noop'`: return the original input unchanged (skips processing)
73
73
  * @param {boolean} [normalizeSpaces=false] - Replace look-alike spaces with an ordinary space. Runs first,
74
74
  * so `purgeInvisibleChars` no longer joins the words around them.
@@ -1,2 +1,2 @@
1
1
  /** The installed version of text-sanctifier, as in its package.json. */
2
- export const VERSION: "1.3.9";
2
+ export const VERSION: "1.3.11";
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "text-sanctifier",
3
- "version": "1.3.9",
3
+ "version": "1.3.11",
4
4
  "type": "module",
5
- "description": "A brutal text normalizer and invisible trash scrubber for modern web projects.",
5
+ "description": "Find and remove invisible Unicode in web input, model-bound text and repositories.",
6
6
  "engines": {
7
7
  "node": ">=18"
8
8
  },
@@ -48,15 +48,17 @@
48
48
  "build": "node scripts/build.js",
49
49
  "build:types": "tsc -p tsconfig.types.json",
50
50
  "test:types": "tsc -p tests/types/tsconfig.json",
51
- "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js && node tests/release.test.js",
51
+ "test": "node tests/sanctifyText.test.js && node tests/inspectTextDetailed.test.js && node --expose-gc tests/hiddenText.test.js && node --expose-gc tests/context.test.js && node tests/cli.test.js && node tests/release.test.js && node tests/names.test.js && node tests/docs.test.js && node tests/repo.test.js",
52
52
  "test-min": "node tests/sanctifyText.test.min.js",
53
53
  "check:self": "node bin/text-sanctifier.js check .",
54
54
  "check:names": "node scripts/check-names.js",
55
- "ci": "npm run gen:check && npm run check:names && npm run check:self && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
55
+ "check:repo": "node scripts/check-repo.js",
56
+ "ci": "npm run gen:check && npm run check:names && npm run check:repo && npm run check:self && npm run test && npm run build && npm run size && npm run build:types && npm run test:types && npm run test-min",
56
57
  "size": "node scripts/check-size.js",
57
58
  "bench": "node --expose-gc scripts/bench.js",
58
59
  "release:prep": "node scripts/release.js prep",
59
60
  "release:verify": "node scripts/release.js verify",
61
+ "prepublishOnly": "node scripts/release.js check-publish",
60
62
  "hooks:install": "git config core.hooksPath scripts/hooks"
61
63
  },
62
64
  "keywords": [
@@ -83,7 +85,7 @@
83
85
  "test": "tests"
84
86
  },
85
87
  "devDependencies": {
86
- "esbuild": "^0.25.3",
87
- "typescript": "^5.9.3"
88
+ "esbuild": "0.25.12",
89
+ "typescript": "5.9.3"
88
90
  }
89
91
  }
@@ -0,0 +1,171 @@
1
+ // Internal hidden-span scanner; not a package entry point.
2
+
3
+ /**
4
+ * @typedef {import('./hiddenText.js').HiddenRunKind} HiddenRunKind
5
+ * @typedef {import('./hiddenText.js').HiddenRun} HiddenRun
6
+ * @typedef {import('./hiddenText.js').HiddenRunCounts} HiddenRunCounts
7
+ * @typedef {import('./hiddenText.js').HiddenTextScan} HiddenTextScan
8
+ * @typedef {import('./hiddenText.js').HiddenScanOptions} HiddenScanOptions
9
+ */
10
+
11
+ const TAG_FIRST = 0xE0000;
12
+ const TAG_LAST = 0xE007F;
13
+ const CANCEL_TAG = 0xE007F;
14
+ const BLACK_FLAG = 0x1F3F4;
15
+
16
+ // RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
17
+ const SUBDIVISION_FLAGS = new Set(['gbeng', 'gbsct', 'gbwls']);
18
+
19
+ /**
20
+ * @param {number} cp
21
+ * @returns {HiddenRunKind | null}
22
+ */
23
+ function kindOf(cp) {
24
+ if (cp >= TAG_FIRST && cp <= TAG_LAST) return 'tag';
25
+ if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
26
+ (cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
27
+ if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
28
+ // Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
29
+ if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
30
+ return null;
31
+ }
32
+
33
+ /** @param {number} cp */
34
+ const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
35
+
36
+ /**
37
+ * Decodes text[start, end) as decodeTagText does, stopping before the output would
38
+ * pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
39
+ * @param {string} text
40
+ * @param {number} start
41
+ * @param {number} end
42
+ * @param {number} limit
43
+ * @returns {{ decoded: string, truncated: boolean }}
44
+ */
45
+ export function decodeRange(text, start, end, limit) {
46
+ let out = '';
47
+ let i = start;
48
+ while (i < end) {
49
+ const cp = /** @type {number} */ (text.codePointAt(i));
50
+ const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
51
+ if (out.length + piece.length > limit) return { decoded: out, truncated: true };
52
+ out += piece;
53
+ i += cp > 0xFFFF ? 2 : 1;
54
+ }
55
+ return { decoded: out, truncated: false };
56
+ }
57
+
58
+ // Tested at an index of the text itself, so the check allocates nothing.
59
+ const PICTOGRAPHIC_AT = /\p{Extended_Pictographic}/yu;
60
+
61
+ /**
62
+ * True when a presentation selector at `index` follows an emoji-capable character:
63
+ * an Extended_Pictographic code point, or a keycap base (#, * or a digit) with U+20E3 after.
64
+ * @param {string} text
65
+ * @param {number} index
66
+ * @returns {boolean}
67
+ */
68
+ function isPresentationBase(text, index) {
69
+ if (index === 0) return false;
70
+ const low = text.charCodeAt(index - 1);
71
+ const p = low >= 0xDC00 && low <= 0xDFFF && index >= 2 &&
72
+ (text.charCodeAt(index - 2) & 0xFC00) === 0xD800 ? index - 2 : index - 1;
73
+ const before = text.charCodeAt(p);
74
+ if ((before === 0x23 || before === 0x2A || (before >= 0x30 && before <= 0x39)) && text.charCodeAt(index + 1) === 0x20E3) return true;
75
+ PICTOGRAPHIC_AT.lastIndex = p;
76
+ return PICTOGRAPHIC_AT.test(text);
77
+ }
78
+
79
+ /**
80
+ * A cap option: a non-negative integer or Infinity; anything else means no cap.
81
+ * @param {unknown} value
82
+ * @returns {number}
83
+ */
84
+ function cap(value) {
85
+ return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
86
+ ? value
87
+ : Infinity;
88
+ }
89
+
90
+ /**
91
+ * @param {string} text
92
+ * @param {HiddenScanOptions} [options]
93
+ * @param {(span: {offset: number, end: number, benign: boolean, kind: HiddenRunKind}) => void} [onSpan] - internal streaming visitor
94
+ * @returns {HiddenTextScan}
95
+ */
96
+ export function scanHiddenSpans(text, options = {}, onSpan) {
97
+ if (typeof text !== 'string') throw new TypeError('findHiddenRuns expects a string input.');
98
+ const maxRuns = cap(options && options.maxRuns);
99
+ const maxDecodedLength = cap(options && options.maxDecodedLength);
100
+
101
+ /** @type {HiddenRun[]} */
102
+ const runs = [];
103
+ /** @type {Array<[number, number]>} */
104
+ const benign = [];
105
+ /** @type {HiddenRunCounts} */
106
+ const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
107
+ /** @type {HiddenRunCounts} */
108
+ const codePointCounts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
109
+ let supplementarySelectorCount = 0;
110
+ let total = 0;
111
+ let benignTotal = 0;
112
+ let line = 1;
113
+ let lineStart = 0;
114
+
115
+ let i = 0;
116
+ while (i < text.length) {
117
+ const cp = /** @type {number} */ (text.codePointAt(i));
118
+ const kind = kindOf(cp);
119
+ if (kind === null) {
120
+ if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
121
+ line++;
122
+ lineStart = i + 1;
123
+ }
124
+ i += cp > 0xFFFF ? 2 : 1;
125
+ continue;
126
+ }
127
+
128
+ const offset = i;
129
+ let count = 0;
130
+ let supplementary = 0;
131
+ let last = cp;
132
+ while (i < text.length) {
133
+ const c = /** @type {number} */ (text.codePointAt(i));
134
+ if (kindOf(c) !== kind) break;
135
+ last = c;
136
+ count++;
137
+ if (c >= 0xE0100 && c <= 0xE01EF) supplementary++;
138
+ i += c > 0xFFFF ? 2 : 1;
139
+ }
140
+ const end = i;
141
+
142
+ // A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
143
+ const isBenign = kind === 'tag'
144
+ ? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
145
+ SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
146
+ : kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
147
+ isPresentationBase(text, offset);
148
+ if (onSpan) onSpan({ offset, end, benign: isBenign, kind });
149
+ if (isBenign) {
150
+ benignTotal++;
151
+ if (benign.length < maxRuns) benign.push([offset, end]);
152
+ continue;
153
+ }
154
+
155
+ counts[kind]++;
156
+ codePointCounts[kind] += count;
157
+ supplementarySelectorCount += supplementary;
158
+ total++;
159
+ if (runs.length < maxRuns) {
160
+ /** @type {HiddenRun} */
161
+ const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
162
+ if (kind === 'tag') {
163
+ const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
164
+ run.decoded = decoded;
165
+ run.decodedTruncated = truncated;
166
+ }
167
+ runs.push(run);
168
+ }
169
+ }
170
+ return { runs, benign, counts, codePointCounts, supplementarySelectorCount, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
171
+ }
package/src/hiddenText.js CHANGED
@@ -9,6 +9,8 @@
9
9
  // Plain JavaScript with no Node APIs, so it also runs in a browser. src/index.js does
10
10
  // not import this file, so it stays out of the browser bundle and its size budget.
11
11
 
12
+ import { scanHiddenSpans, decodeRange } from './hiddenScan.js';
13
+
12
14
  export { VERSION } from './version.generated.js';
13
15
 
14
16
  /** @typedef {'tag' | 'variation_selector' | 'invisible_operator' | 'reserved_tag_plane'} HiddenRunKind */
@@ -56,31 +58,6 @@ export { VERSION } from './version.generated.js';
56
58
  * For both, any other value (negative, fractional, NaN, a string) means the default, no cap.
57
59
  */
58
60
 
59
- const TAG_FIRST = 0xE0000;
60
- const TAG_LAST = 0xE007F;
61
- const CANCEL_TAG = 0xE007F;
62
- const BLACK_FLAG = 0x1F3F4;
63
-
64
- // RGI subdivision flags: U+1F3F4, the tag letters, U+E007F
65
- const SUBDIVISION_FLAGS = new Set(['gbeng', 'gbsct', 'gbwls']);
66
-
67
- /**
68
- * @param {number} cp
69
- * @returns {HiddenRunKind | null}
70
- */
71
- function kindOf(cp) {
72
- if (cp >= TAG_FIRST && cp <= TAG_LAST) return 'tag';
73
- if ((cp >= 0xFE00 && cp <= 0xFE0F) || (cp >= 0xE0100 && cp <= 0xE01EF) ||
74
- (cp >= 0x180B && cp <= 0x180D) || cp === 0x180F) return 'variation_selector';
75
- if (cp >= 0x2061 && cp <= 0x2064) return 'invisible_operator';
76
- // Unassigned and default ignorable: U+E0080-U+E00FF and U+E01F0-U+E0FFF
77
- if (cp >= 0xE0080 && cp <= 0xE0FFF) return 'reserved_tag_plane';
78
- return null;
79
- }
80
-
81
- /** @param {number} cp */
82
- const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
83
-
84
61
  /**
85
62
  * Decodes tag characters to the ASCII they shadow: U+E0020–U+E007E become
86
63
  * U+0020–U+007E. Anything else (U+E0001 LANGUAGE TAG, U+E007F CANCEL TAG, unassigned
@@ -89,63 +66,10 @@ const hex = (cp) => 'U+' + cp.toString(16).toUpperCase().padStart(4, '0');
89
66
  * @returns {string}
90
67
  */
91
68
  export function decodeTagText(text) {
69
+ if (typeof text !== 'string') throw new TypeError('decodeTagText expects a string input.');
92
70
  return decodeRange(text, 0, text.length, Infinity).decoded;
93
71
  }
94
72
 
95
- /**
96
- * Decodes text[start, end) as decodeTagText does, stopping before the output would
97
- * pass `limit` UTF-16 units. A `<U+XXXX>` marker is written whole or not at all.
98
- * @param {string} text
99
- * @param {number} start
100
- * @param {number} end
101
- * @param {number} limit
102
- * @returns {{ decoded: string, truncated: boolean }}
103
- */
104
- function decodeRange(text, start, end, limit) {
105
- let out = '';
106
- let i = start;
107
- while (i < end) {
108
- const cp = /** @type {number} */ (text.codePointAt(i));
109
- const piece = cp >= 0xE0020 && cp <= 0xE007E ? String.fromCharCode(cp - TAG_FIRST) : `<${hex(cp)}>`;
110
- if (out.length + piece.length > limit) return { decoded: out, truncated: true };
111
- out += piece;
112
- i += cp > 0xFFFF ? 2 : 1;
113
- }
114
- return { decoded: out, truncated: false };
115
- }
116
-
117
- // Tested at an index of the text itself, so the check allocates nothing.
118
- const PICTOGRAPHIC_AT = /\p{Extended_Pictographic}/yu;
119
-
120
- /**
121
- * True when a presentation selector at `index` follows an emoji-capable character:
122
- * an Extended_Pictographic code point, or a keycap base (#, * or a digit) with U+20E3 after.
123
- * @param {string} text
124
- * @param {number} index
125
- * @returns {boolean}
126
- */
127
- function isPresentationBase(text, index) {
128
- if (index === 0) return false;
129
- const low = text.charCodeAt(index - 1);
130
- const p = low >= 0xDC00 && low <= 0xDFFF && index >= 2 &&
131
- (text.charCodeAt(index - 2) & 0xFC00) === 0xD800 ? index - 2 : index - 1;
132
- const before = text.charCodeAt(p);
133
- if ((before === 0x23 || before === 0x2A || (before >= 0x30 && before <= 0x39)) && text.charCodeAt(index + 1) === 0x20E3) return true;
134
- PICTOGRAPHIC_AT.lastIndex = p;
135
- return PICTOGRAPHIC_AT.test(text);
136
- }
137
-
138
- /**
139
- * A cap option: a non-negative integer or Infinity; anything else means no cap.
140
- * @param {unknown} value
141
- * @returns {number}
142
- */
143
- function cap(value) {
144
- return typeof value === 'number' && (value === Infinity || (Number.isInteger(value) && value >= 0))
145
- ? value
146
- : Infinity;
147
- }
148
-
149
73
  /**
150
74
  * Finds runs of tag characters, variation selectors, invisible operators and reserved
151
75
  * tag-plane code points (U+E0080–U+E00FF, U+E01F0–U+E0FFF).
@@ -168,76 +92,5 @@ function cap(value) {
168
92
  * @returns {HiddenTextScan}
169
93
  */
170
94
  export function findHiddenRuns(text, options = {}) {
171
- const maxRuns = cap(options && options.maxRuns);
172
- const maxDecodedLength = cap(options && options.maxDecodedLength);
173
-
174
- /** @type {HiddenRun[]} */
175
- const runs = [];
176
- /** @type {Array<[number, number]>} */
177
- const benign = [];
178
- /** @type {HiddenRunCounts} */
179
- const counts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
180
- /** @type {HiddenRunCounts} */
181
- const codePointCounts = { tag: 0, variation_selector: 0, invisible_operator: 0, reserved_tag_plane: 0 };
182
- let supplementarySelectorCount = 0;
183
- let total = 0;
184
- let benignTotal = 0;
185
- let line = 1;
186
- let lineStart = 0;
187
-
188
- let i = 0;
189
- while (i < text.length) {
190
- const cp = /** @type {number} */ (text.codePointAt(i));
191
- const kind = kindOf(cp);
192
- if (kind === null) {
193
- if (cp === 0x0A || (cp === 0x0D && text.charCodeAt(i + 1) !== 0x0A)) { // CRLF ends at its LF
194
- line++;
195
- lineStart = i + 1;
196
- }
197
- i += cp > 0xFFFF ? 2 : 1;
198
- continue;
199
- }
200
-
201
- const offset = i;
202
- let count = 0;
203
- let supplementary = 0;
204
- let last = cp;
205
- while (i < text.length) {
206
- const c = /** @type {number} */ (text.codePointAt(i));
207
- if (kindOf(c) !== kind) break;
208
- last = c;
209
- count++;
210
- if (c >= 0xE0100 && c <= 0xE01EF) supplementary++;
211
- i += c > 0xFFFF ? 2 : 1;
212
- }
213
- const end = i;
214
-
215
- // A subdivision flag is exactly five tag letters and CANCEL TAG after U+1F3F4.
216
- const isBenign = kind === 'tag'
217
- ? count === 6 && last === CANCEL_TAG && offset >= 2 && text.codePointAt(offset - 2) === BLACK_FLAG &&
218
- SUBDIVISION_FLAGS.has(decodeRange(text, offset, end - 2, Infinity).decoded)
219
- : kind === 'variation_selector' && count === 1 && (cp === 0xFE0E || cp === 0xFE0F) &&
220
- isPresentationBase(text, offset);
221
- if (isBenign) {
222
- benignTotal++;
223
- if (benign.length < maxRuns) benign.push([offset, end]);
224
- continue;
225
- }
226
-
227
- counts[kind]++;
228
- codePointCounts[kind] += count;
229
- supplementarySelectorCount += supplementary;
230
- total++;
231
- if (runs.length < maxRuns) {
232
- /** @type {HiddenRun} */
233
- const run = { kind, offset, end, line, column: offset - lineStart + 1, count };
234
- if (kind === 'tag') {
235
- const { decoded, truncated } = decodeRange(text, offset, end, maxDecodedLength);
236
- run.decoded = decoded;
237
- run.decodedTruncated = truncated;
238
- }
239
- runs.push(run);
240
- }
241
- }
242
- return { runs, benign, counts, codePointCounts, supplementarySelectorCount, total, benignTotal, truncated: total > runs.length || benignTotal > benign.length };
95
+ return scanHiddenSpans(text, options);
243
96
  }
@@ -322,16 +322,20 @@ export function getNewlineStats(text) {
322
322
  }
323
323
 
324
324
  /**
325
- * Creates defaultOptions for summonSanctifier based on inspectText result
325
+ * Creates defaultOptions for summonSanctifier based on inspectText result.
326
+ * Recommends only what removes trash without losing meaning: non-ASCII text and emoji are
327
+ * content, so `purgeEmojis` and `keyboardOnlyFilter` stay opt-in, and look-alike spaces
328
+ * become ordinary spaces so that words do not run together.
326
329
  * @param {!UnicodeTrashReport} report
327
330
  * @return {!SanctifyOptions}
328
331
  */
329
332
  export function getRecommendedSanctifierOptions(report) {
330
333
  return {
334
+ normalizeSpaces: report.hasInvisibleChars,
331
335
  purgeInvisibleChars: report.hasInvisibleChars,
332
- purgeEmojis: report.hasEmojis,
336
+ purgeEmojis: false,
333
337
  nukeControls: report.hasControlChars,
334
- keyboardOnlyFilter: report.hasNonKeyboardChars,
338
+ keyboardOnlyFilter: false,
335
339
  normalizeNewlines:
336
340
  report.hasMixedNewlines || report.newlineStyle === 'CRLF' || report.newlineStyle === 'CR',
337
341
  };
@@ -294,7 +294,7 @@ export const summonSanctifier = /* @__PURE__ */ Object.assign(createSanctifier,
294
294
  * @param {number} [maxLength=Infinity] - Hard input length cap (UTF-16 code units via `text.length`).
295
295
  * @param {'throw'|'truncate'|'noop'} [onMaxLength='throw'] - Behavior when `text.length` exceeds `maxLength`:
296
296
  * - `'throw'`: throw a `RangeError`
297
- * - `'truncate'`: slice to `maxLength` before processing
297
+ * - `'truncate'`: slice to at most `maxLength` before processing, never through a surrogate pair
298
298
  * - `'noop'`: return the original input unchanged (skips processing)
299
299
  * @param {boolean} [normalizeSpaces=false] - Replace look-alike spaces with an ordinary space. Runs first,
300
300
  * so `purgeInvisibleChars` no longer joins the words around them.
@@ -324,26 +324,27 @@ export function sanctifyText(
324
324
  throw new TypeError('sanctifyText expects a string input.');
325
325
  }
326
326
 
327
- if (onMaxLength !== 'truncate' && onMaxLength !== 'noop' && onMaxLength !== 'throw') {
328
- onMaxLength = 'throw';
329
- }
330
-
331
327
  if (text.length > maxLength) {
332
328
  switch (onMaxLength) {
333
329
  case 'truncate':
334
- text = text.slice(0, maxLength);
330
+ {
331
+ // Cut before a surrogate pair, not through it
332
+ const last = text.charCodeAt(maxLength - 1);
333
+ text = text.slice(0, last >= 0xD800 && last <= 0xDBFF ? maxLength - 1 : maxLength);
334
+ }
335
335
  break;
336
336
  case 'noop':
337
337
  return text;
338
- case 'throw':
339
- default:
338
+ default: // 'throw', and any invalid value
340
339
  throw new RangeError(
341
340
  `sanctifyText input length ${text.length} exceeds maxLength ${maxLength}.`
342
341
  );
343
342
  }
344
343
  }
345
344
 
346
- let cleaned = text;
345
+ // Unpaired surrogates become U+FFFD first, so removing a character between two
346
+ // halves cannot join them into a new character, such as a hidden tag.
347
+ let cleaned = text.replace(LONE_SURROGATES_REGEX, '\uFFFD');
347
348
 
348
349
  // Look-alike spaces become ordinary spaces, before anything can delete them
349
350
  if (normalizeSpaces) cleaned = cleaned.replace(SPACE_LOOKALIKES_REGEX, ' ');
@@ -389,6 +390,9 @@ export function sanctifyText(
389
390
  */
390
391
  export const SPACE_LOOKALIKES_REGEX = /[\u00A0\u2000-\u200A\u202F\u205F]/g;
391
392
 
393
+ /** With the `u` flag a surrogate pair is one character, so this matches only unpaired halves. */
394
+ const LONE_SURROGATES_REGEX = /[\uD800-\uDFFF]/gu;
395
+
392
396
  /** Zero-width space, word joiner and BOM: invisible and never needed in plain text. */
393
397
  export const ZERO_WIDTH_SPACES_REGEX = /[\u200B\u2060\uFEFF]/g;
394
398
 
@@ -497,9 +501,11 @@ function normalizeNewlineChars(text, normalized = '\n') {
497
501
  * @param {string} text
498
502
  * @returns {string}
499
503
  */
500
- const TRIM_SPACES_AROUND_NEWLINES_REGEX = /[ \t]*(\n+)[ \t]*/g;
504
+ // The second branch consumes a run with no newline after it, so matching resumes past the
505
+ // run instead of retrying from each of its characters, which took quadratic time.
506
+ const TRIM_SPACES_AROUND_NEWLINES_REGEX = /[ \t]*(\n+)[ \t]*|([ \t]+)/g;
501
507
  function trimSpacesAroundNewlineChars(text) {
502
- return text.replace(TRIM_SPACES_AROUND_NEWLINES_REGEX, '$1');
508
+ return text.replace(TRIM_SPACES_AROUND_NEWLINES_REGEX, '$1$2');
503
509
  }
504
510
 
505
511
 
@@ -2,4 +2,4 @@
2
2
  // Generated from package.json by scripts/gen-version.js (`npm run gen`); do not edit.
3
3
 
4
4
  /** The installed version of text-sanctifier, as in its package.json. */
5
- export const VERSION = '1.3.9';
5
+ export const VERSION = '1.3.11';
package/src/index.d.ts DELETED
@@ -1,22 +0,0 @@
1
- import { inspectText } from './inspectText.js';
2
- import { inspectTextDetailed } from './inspectText.js';
3
- import { getNewlineStats } from './inspectText.js';
4
- import { getRecommendedSanctifierOptions } from './inspectText.js';
5
- import { summonSanctifier } from './sanctifyText.js';
6
- import { sanctifyStrict } from './sanctifyText.js';
7
- import { sanctifyLoose } from './sanctifyText.js';
8
- import { sanctifyKeyboardOnlyEmoji } from './sanctifyText.js';
9
- import { sanctifyKeyboardOnly } from './sanctifyText.js';
10
- import { sanctifyPasted } from './sanctifyText.js';
11
- import { revealInvisible } from './reveal.js';
12
- import { VERSION } from './version.generated.js';
13
- export { inspectText, inspectTextDetailed, getNewlineStats, getRecommendedSanctifierOptions, summonSanctifier, sanctifyStrict, sanctifyLoose, sanctifyKeyboardOnlyEmoji, sanctifyKeyboardOnly, sanctifyPasted, revealInvisible, VERSION };
14
- export type UnicodeTrashReport = import('./inspectText.js').UnicodeTrashReport;
15
- export type TrashCategory = import('./inspectText.js').TrashCategory;
16
- export type TrashFinding = import('./inspectText.js').TrashFinding;
17
- export type TrashCounts = import('./inspectText.js').TrashCounts;
18
- export type DetailedTrashReport = import('./inspectText.js').DetailedTrashReport;
19
- export type InspectDetailedOptions = import('./inspectText.js').InspectDetailedOptions;
20
- export type NewlineStats = import('./inspectText.js').NewlineStats;
21
- export type SanctifyOptions = import('./sanctifyText.js').SanctifyOptions;
22
- export type RevealOptions = import('./reveal.js').RevealOptions;