rapier-markdown-kit 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +40 -0
- package/README.md +102 -0
- package/dist/agent/vendor/pretext/LICENSE +21 -0
- package/dist/agent/vendor/pretext/NOTICE +15 -0
- package/dist/agent/vendor/pretext/SOURCE.json +85 -0
- package/dist/agent/vendor/pretext/analysis.js +1234 -0
- package/dist/agent/vendor/pretext/bidi.js +151 -0
- package/dist/agent/vendor/pretext/generated/bidi-data.js +3831 -0
- package/dist/agent/vendor/pretext/layout.js +308 -0
- package/dist/agent/vendor/pretext/line-break.js +636 -0
- package/dist/agent/vendor/pretext/line-text.js +44 -0
- package/dist/agent/vendor/pretext/measurement.js +189 -0
- package/dist/agent/vendor/pretext/package.json +3 -0
- package/dist/agent/vendor/pretext/rich-inline.js +291 -0
- package/dist/agent/will.mjs +167 -0
- package/dist/kit/assets.mjs +3 -0
- package/dist/kit/conformance/01-inline.expected.json +82 -0
- package/dist/kit/conformance/01-inline.md +10 -0
- package/dist/kit/conformance/02-width-x-align.expected.json +83 -0
- package/dist/kit/conformance/02-width-x-align.md +12 -0
- package/dist/kit/conformance/03-wrap-around-silhouette.expected.json +177 -0
- package/dist/kit/conformance/03-wrap-around-silhouette.md +12 -0
- package/dist/kit/conformance/04-wrap-box.expected.json +173 -0
- package/dist/kit/conformance/04-wrap-box.md +12 -0
- package/dist/kit/conformance/05-behind.expected.json +122 -0
- package/dist/kit/conformance/05-behind.md +12 -0
- package/dist/kit/conformance/06-front.expected.json +122 -0
- package/dist/kit/conformance/06-front.md +12 -0
- package/dist/kit/conformance/07-rotate-raster.expected.json +179 -0
- package/dist/kit/conformance/07-rotate-raster.md +12 -0
- package/dist/kit/conformance/08-drawing-rotation.expected.json +188 -0
- package/dist/kit/conformance/08-drawing-rotation.md +12 -0
- package/dist/kit/conformance/09-drawing-ring-interior.expected.json +229 -0
- package/dist/kit/conformance/09-drawing-ring-interior.md +12 -0
- package/dist/kit/conformance/10-both-sides.expected.json +221 -0
- package/dist/kit/conformance/10-both-sides.md +10 -0
- package/dist/kit/conformance/11-neighbour-skips-image.expected.json +160 -0
- package/dist/kit/conformance/11-neighbour-skips-image.md +13 -0
- package/dist/kit/conformance/12-heading-barrier.expected.json +164 -0
- package/dist/kit/conformance/12-heading-barrier.md +12 -0
- package/dist/kit/conformance/13-rtl-text.expected.json +179 -0
- package/dist/kit/conformance/13-rtl-text.md +8 -0
- package/dist/kit/conformance/README.md +99 -0
- package/dist/kit/conformance/measurer.mjs +0 -0
- package/dist/kit/conformance/run.mjs +123 -0
- package/dist/kit/index.mjs +6 -0
- package/dist/kit/layout.mjs +2 -0
- package/dist/kit/marks.mjs +2 -0
- package/dist/kit/model.mjs +3 -0
- package/dist/kit/will.mjs +2 -0
- package/dist/layout/model.mjs +364 -0
- package/dist/spec/md-assets.mjs +323 -0
- package/dist/spec/md-layout.mjs +117 -0
- package/dist/spec/md-marks.mjs +78 -0
- package/package.json +33 -0
|
@@ -0,0 +1,1234 @@
|
|
|
1
|
+
/*! Pretext 0.0.9 | MIT | Copyright (c) 2026 Pretext contributors */
|
|
2
|
+
const collapsibleWhitespaceRunRe = /[ \t\n\r\f]+/g;
|
|
3
|
+
const needsWhitespaceNormalizationRe = /[\t\n\r\f]| {2,}|^ | $/;
|
|
4
|
+
export function normalizeWhitespaceNormal(text) {
|
|
5
|
+
if (!needsWhitespaceNormalizationRe.test(text)) return text;
|
|
6
|
+
let normalized = text.replace(collapsibleWhitespaceRunRe, ' ');
|
|
7
|
+
if (normalized.charCodeAt(0) === 0x20) {
|
|
8
|
+
normalized = normalized.slice(1);
|
|
9
|
+
}
|
|
10
|
+
if (normalized.length > 0 && normalized.charCodeAt(normalized.length - 1) === 0x20) {
|
|
11
|
+
normalized = normalized.slice(0, -1);
|
|
12
|
+
}
|
|
13
|
+
return normalized;
|
|
14
|
+
}
|
|
15
|
+
function normalizeWhitespacePreWrap(text) {
|
|
16
|
+
if (!/[\r\f]/.test(text)) return text;
|
|
17
|
+
return text.replace(/\r\n/g, '\n').replace(/[\r\f]/g, '\n');
|
|
18
|
+
}
|
|
19
|
+
let sharedGraphemeSegmenter = null;
|
|
20
|
+
export function getSharedGraphemeSegmenter() {
|
|
21
|
+
if (sharedGraphemeSegmenter === null) {
|
|
22
|
+
sharedGraphemeSegmenter = new Intl.Segmenter(undefined, {
|
|
23
|
+
granularity: 'grapheme'
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
return sharedGraphemeSegmenter;
|
|
27
|
+
}
|
|
28
|
+
let sharedWordSegmenter = null;
|
|
29
|
+
let segmenterLocale;
|
|
30
|
+
function getSharedWordSegmenter() {
|
|
31
|
+
if (sharedWordSegmenter === null) {
|
|
32
|
+
sharedWordSegmenter = new Intl.Segmenter(segmenterLocale, {
|
|
33
|
+
granularity: 'word'
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
return sharedWordSegmenter;
|
|
37
|
+
}
|
|
38
|
+
export function clearAnalysisCaches() {
|
|
39
|
+
sharedGraphemeSegmenter = null;
|
|
40
|
+
sharedWordSegmenter = null;
|
|
41
|
+
}
|
|
42
|
+
export function setAnalysisLocale(locale) {
|
|
43
|
+
const nextLocale = locale && locale.length > 0 ? locale : undefined;
|
|
44
|
+
if (segmenterLocale === nextLocale) return;
|
|
45
|
+
segmenterLocale = nextLocale;
|
|
46
|
+
sharedWordSegmenter = null;
|
|
47
|
+
}
|
|
48
|
+
const arabicScriptRe = /\p{Script=Arabic}/u;
|
|
49
|
+
const combiningMarkRe = /\p{M}/u;
|
|
50
|
+
const decimalDigitRe = /\p{Nd}/u;
|
|
51
|
+
function containsArabicScript(text) {
|
|
52
|
+
return arabicScriptRe.test(text);
|
|
53
|
+
}
|
|
54
|
+
const cjkRe = /[\u3000-\u30FF\u3130-\u318F\u3400-\u4DBF\u4E00-\u9FFF\uAC00-\uD7AF\uF900-\uFAFF\uFF00-\uFFEF\u{20000}-\u{2A6DF}\u{2A700}-\u{2EE5D}\u{2F800}-\u{2FA1F}\u{30000}-\u{33479}]/u;
|
|
55
|
+
export function isCJK(s) {
|
|
56
|
+
return cjkRe.test(s);
|
|
57
|
+
}
|
|
58
|
+
function endsWithLineStartProhibitedText(text) {
|
|
59
|
+
const last = getLastCodePoint(text);
|
|
60
|
+
return last !== null && (kinsokuStart.has(last) || leftStickyPunctuation.has(last));
|
|
61
|
+
}
|
|
62
|
+
const keepAllGlueChars = new Set([
|
|
63
|
+
'\u00A0',
|
|
64
|
+
'\u202F',
|
|
65
|
+
'\u2060',
|
|
66
|
+
'\uFEFF'
|
|
67
|
+
]);
|
|
68
|
+
const keepAllDashBreakChars = new Set([
|
|
69
|
+
'-',
|
|
70
|
+
'\u2010',
|
|
71
|
+
'\u2013',
|
|
72
|
+
'\u2014'
|
|
73
|
+
]);
|
|
74
|
+
function endsWithKeepAllGlueText(text) {
|
|
75
|
+
const last = getLastCodePoint(text);
|
|
76
|
+
return last !== null && keepAllGlueChars.has(last);
|
|
77
|
+
}
|
|
78
|
+
function endsWithKeepAllDashBreakText(text) {
|
|
79
|
+
const last = getLastCodePoint(text);
|
|
80
|
+
return last !== null && keepAllDashBreakChars.has(last);
|
|
81
|
+
}
|
|
82
|
+
export function canContinueKeepAllTextRun(previousText, breakAfterPunctuation) {
|
|
83
|
+
if (endsWithKeepAllGlueText(previousText)) return false;
|
|
84
|
+
if (!breakAfterPunctuation) return true;
|
|
85
|
+
if (endsWithLineStartProhibitedText(previousText)) return false;
|
|
86
|
+
if (endsWithKeepAllDashBreakText(previousText)) return false;
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
export const kinsokuStart = new Set([
|
|
90
|
+
'\uFF0C',
|
|
91
|
+
'\uFF0E',
|
|
92
|
+
'\uFF01',
|
|
93
|
+
'\uFF1A',
|
|
94
|
+
'\uFF1B',
|
|
95
|
+
'\uFF1F',
|
|
96
|
+
'\u3001',
|
|
97
|
+
'\u3002',
|
|
98
|
+
'\u30FB',
|
|
99
|
+
'\uFF09',
|
|
100
|
+
'\u3015',
|
|
101
|
+
'\u3009',
|
|
102
|
+
'\u300B',
|
|
103
|
+
'\u300D',
|
|
104
|
+
'\u300F',
|
|
105
|
+
'\u3011',
|
|
106
|
+
'\u3017',
|
|
107
|
+
'\u3019',
|
|
108
|
+
'\u301B',
|
|
109
|
+
'\u30FC',
|
|
110
|
+
'\u3005',
|
|
111
|
+
'\u303B',
|
|
112
|
+
'\u309D',
|
|
113
|
+
'\u309E',
|
|
114
|
+
'\u30FD',
|
|
115
|
+
'\u30FE'
|
|
116
|
+
]);
|
|
117
|
+
export const kinsokuEnd = new Set([
|
|
118
|
+
'"',
|
|
119
|
+
'(',
|
|
120
|
+
'[',
|
|
121
|
+
'{',
|
|
122
|
+
'¡',
|
|
123
|
+
'¿',
|
|
124
|
+
'“',
|
|
125
|
+
'‘',
|
|
126
|
+
'‚',
|
|
127
|
+
'„',
|
|
128
|
+
'«',
|
|
129
|
+
'‹',
|
|
130
|
+
'\u2E18',
|
|
131
|
+
'\uFF08',
|
|
132
|
+
'\u3014',
|
|
133
|
+
'\u3008',
|
|
134
|
+
'\u300A',
|
|
135
|
+
'\u300C',
|
|
136
|
+
'\u300E',
|
|
137
|
+
'\u3010',
|
|
138
|
+
'\u3016',
|
|
139
|
+
'\u3018',
|
|
140
|
+
'\u301A'
|
|
141
|
+
]);
|
|
142
|
+
const forwardStickyGlue = new Set([
|
|
143
|
+
"'",
|
|
144
|
+
'’'
|
|
145
|
+
]);
|
|
146
|
+
export const leftStickyPunctuation = new Set([
|
|
147
|
+
'.',
|
|
148
|
+
',',
|
|
149
|
+
'!',
|
|
150
|
+
'?',
|
|
151
|
+
':',
|
|
152
|
+
';',
|
|
153
|
+
'\u060C',
|
|
154
|
+
'\u061B',
|
|
155
|
+
'\u061F',
|
|
156
|
+
'\u0964',
|
|
157
|
+
'\u0965',
|
|
158
|
+
'\u104A',
|
|
159
|
+
'\u104B',
|
|
160
|
+
'\u104C',
|
|
161
|
+
'\u104D',
|
|
162
|
+
'\u104F',
|
|
163
|
+
')',
|
|
164
|
+
']',
|
|
165
|
+
'}',
|
|
166
|
+
'%',
|
|
167
|
+
'"',
|
|
168
|
+
'”',
|
|
169
|
+
'’',
|
|
170
|
+
'»',
|
|
171
|
+
'›',
|
|
172
|
+
'…'
|
|
173
|
+
]);
|
|
174
|
+
const arabicNoSpaceTrailingPunctuation = new Set([
|
|
175
|
+
':',
|
|
176
|
+
'.',
|
|
177
|
+
'\u060C',
|
|
178
|
+
'\u061B'
|
|
179
|
+
]);
|
|
180
|
+
const myanmarMedialGlue = new Set([
|
|
181
|
+
'\u104F'
|
|
182
|
+
]);
|
|
183
|
+
const closingQuoteChars = new Set([
|
|
184
|
+
'”',
|
|
185
|
+
'’',
|
|
186
|
+
'»',
|
|
187
|
+
'›',
|
|
188
|
+
'\u300D',
|
|
189
|
+
'\u300F',
|
|
190
|
+
'\u3011',
|
|
191
|
+
'\u300B',
|
|
192
|
+
'\u3009',
|
|
193
|
+
'\u3015',
|
|
194
|
+
'\uFF09'
|
|
195
|
+
]);
|
|
196
|
+
function isLeftStickyPunctuationSegment(segment) {
|
|
197
|
+
if (isPunctuationGlueCluster(segment)) return true;
|
|
198
|
+
let sawPunctuation = false;
|
|
199
|
+
for (const ch of segment){
|
|
200
|
+
if (leftStickyPunctuation.has(ch) || isLineBreakNumericAffix(ch)) {
|
|
201
|
+
sawPunctuation = true;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (sawPunctuation && combiningMarkRe.test(ch)) continue;
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
return sawPunctuation;
|
|
208
|
+
}
|
|
209
|
+
function isCJKLineStartProhibitedSegment(segment) {
|
|
210
|
+
for (const ch of segment){
|
|
211
|
+
if (!kinsokuStart.has(ch) && !leftStickyPunctuation.has(ch)) return false;
|
|
212
|
+
}
|
|
213
|
+
return segment.length > 0;
|
|
214
|
+
}
|
|
215
|
+
function isForwardStickyClusterSegment(segment) {
|
|
216
|
+
if (isPunctuationGlueCluster(segment)) return true;
|
|
217
|
+
for (const ch of segment){
|
|
218
|
+
if (!kinsokuEnd.has(ch) && !forwardStickyGlue.has(ch) && !combiningMarkRe.test(ch) && !isLineBreakNumericAffix(ch)) {
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return segment.length > 0;
|
|
223
|
+
}
|
|
224
|
+
function isPunctuationGlueCluster(segment) {
|
|
225
|
+
let sawPunctuation = false;
|
|
226
|
+
for (const ch of segment){
|
|
227
|
+
if (ch === '\\' || combiningMarkRe.test(ch)) continue;
|
|
228
|
+
if (kinsokuEnd.has(ch) || leftStickyPunctuation.has(ch) || forwardStickyGlue.has(ch)) {
|
|
229
|
+
sawPunctuation = true;
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
return sawPunctuation;
|
|
235
|
+
}
|
|
236
|
+
function previousCodePointStart(text, end) {
|
|
237
|
+
const last = end - 1;
|
|
238
|
+
if (last <= 0) return Math.max(last, 0);
|
|
239
|
+
const lastCodeUnit = text.charCodeAt(last);
|
|
240
|
+
if (lastCodeUnit < 0xDC00 || lastCodeUnit > 0xDFFF) return last;
|
|
241
|
+
const maybeHigh = last - 1;
|
|
242
|
+
if (maybeHigh < 0) return last;
|
|
243
|
+
const highCodeUnit = text.charCodeAt(maybeHigh);
|
|
244
|
+
return highCodeUnit >= 0xD800 && highCodeUnit <= 0xDBFF ? maybeHigh : last;
|
|
245
|
+
}
|
|
246
|
+
function getLastCodePoint(text) {
|
|
247
|
+
if (text.length === 0) return null;
|
|
248
|
+
const start = previousCodePointStart(text, text.length);
|
|
249
|
+
return text.slice(start);
|
|
250
|
+
}
|
|
251
|
+
function getFirstSignificantCodePoint(text) {
|
|
252
|
+
for (const ch of text){
|
|
253
|
+
if (!combiningMarkRe.test(ch)) return ch;
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
function getLastSignificantCodePoint(text, end = text.length) {
|
|
258
|
+
for(; end > 0;){
|
|
259
|
+
const start = previousCodePointStart(text, end);
|
|
260
|
+
const ch = text.slice(start, end);
|
|
261
|
+
if (!combiningMarkRe.test(ch)) return ch;
|
|
262
|
+
end = start;
|
|
263
|
+
}
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
const lineBreakNumericAffixRanges = [
|
|
267
|
+
0x0024,
|
|
268
|
+
0x0025,
|
|
269
|
+
0x002B,
|
|
270
|
+
0x002B,
|
|
271
|
+
0x005C,
|
|
272
|
+
0x005C,
|
|
273
|
+
0x00A2,
|
|
274
|
+
0x00A5,
|
|
275
|
+
0x00B0,
|
|
276
|
+
0x00B1,
|
|
277
|
+
0x058F,
|
|
278
|
+
0x058F,
|
|
279
|
+
0x0609,
|
|
280
|
+
0x060B,
|
|
281
|
+
0x066A,
|
|
282
|
+
0x066A,
|
|
283
|
+
0x07FE,
|
|
284
|
+
0x07FF,
|
|
285
|
+
0x09F2,
|
|
286
|
+
0x09F3,
|
|
287
|
+
0x09F9,
|
|
288
|
+
0x09FB,
|
|
289
|
+
0x0AF1,
|
|
290
|
+
0x0AF1,
|
|
291
|
+
0x0BF9,
|
|
292
|
+
0x0BF9,
|
|
293
|
+
0x0D79,
|
|
294
|
+
0x0D79,
|
|
295
|
+
0x0E3F,
|
|
296
|
+
0x0E3F,
|
|
297
|
+
0x17DB,
|
|
298
|
+
0x17DB,
|
|
299
|
+
0x2030,
|
|
300
|
+
0x2037,
|
|
301
|
+
0x2057,
|
|
302
|
+
0x2057,
|
|
303
|
+
0x20A0,
|
|
304
|
+
0x20CF,
|
|
305
|
+
0x2103,
|
|
306
|
+
0x2103,
|
|
307
|
+
0x2109,
|
|
308
|
+
0x2109,
|
|
309
|
+
0x2116,
|
|
310
|
+
0x2116,
|
|
311
|
+
0x2212,
|
|
312
|
+
0x2213,
|
|
313
|
+
0xA838,
|
|
314
|
+
0xA838,
|
|
315
|
+
0xFDFC,
|
|
316
|
+
0xFDFC,
|
|
317
|
+
0xFE69,
|
|
318
|
+
0xFE6A,
|
|
319
|
+
0xFF04,
|
|
320
|
+
0xFF05,
|
|
321
|
+
0xFFE0,
|
|
322
|
+
0xFFE1,
|
|
323
|
+
0xFFE5,
|
|
324
|
+
0xFFE6,
|
|
325
|
+
0x11FDD,
|
|
326
|
+
0x11FE0,
|
|
327
|
+
0x1E2FF,
|
|
328
|
+
0x1E2FF,
|
|
329
|
+
0x1ECAC,
|
|
330
|
+
0x1ECAC,
|
|
331
|
+
0x1ECB0,
|
|
332
|
+
0x1ECB0
|
|
333
|
+
];
|
|
334
|
+
function isCodePointInRanges(codePoint, ranges) {
|
|
335
|
+
for(let i = 0; i < ranges.length; i += 2){
|
|
336
|
+
if (codePoint >= ranges[i] && codePoint <= ranges[i + 1]) return true;
|
|
337
|
+
}
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
function isLineBreakNumericAffix(ch) {
|
|
341
|
+
const codePoint = ch.codePointAt(0);
|
|
342
|
+
return codePoint !== undefined && isCodePointInRanges(codePoint, lineBreakNumericAffixRanges);
|
|
343
|
+
}
|
|
344
|
+
function endsWithLineBreakNumericAffix(text) {
|
|
345
|
+
const last = getLastSignificantCodePoint(text);
|
|
346
|
+
return last !== null && isLineBreakNumericAffix(last);
|
|
347
|
+
}
|
|
348
|
+
function startsWithDecimalDigit(text) {
|
|
349
|
+
const first = getFirstSignificantCodePoint(text);
|
|
350
|
+
return first !== null && decimalDigitRe.test(first);
|
|
351
|
+
}
|
|
352
|
+
function splitTrailingForwardStickyCluster(text) {
|
|
353
|
+
let splitIndex = text.length;
|
|
354
|
+
while(splitIndex > 0){
|
|
355
|
+
const start = previousCodePointStart(text, splitIndex);
|
|
356
|
+
const ch = text.slice(start, splitIndex);
|
|
357
|
+
if (combiningMarkRe.test(ch) || kinsokuEnd.has(ch) || forwardStickyGlue.has(ch)) {
|
|
358
|
+
splitIndex = start;
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
break;
|
|
362
|
+
}
|
|
363
|
+
if (splitIndex <= 0 || splitIndex === text.length) return null;
|
|
364
|
+
return {
|
|
365
|
+
head: text.slice(0, splitIndex),
|
|
366
|
+
tail: text.slice(splitIndex)
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
function getRepeatableSingleCharRunChar(text, isWordLike, kind) {
|
|
370
|
+
return kind === 'text' && !isWordLike && text.length === 1 && text !== '-' && text !== '—' ? text : null;
|
|
371
|
+
}
|
|
372
|
+
function hasArabicNoSpacePunctuation(containsArabic, lastCodePoint) {
|
|
373
|
+
return containsArabic && lastCodePoint !== null && arabicNoSpaceTrailingPunctuation.has(lastCodePoint);
|
|
374
|
+
}
|
|
375
|
+
function endsWithMyanmarMedialGlue(segment) {
|
|
376
|
+
const lastCodePoint = getLastCodePoint(segment);
|
|
377
|
+
return lastCodePoint !== null && myanmarMedialGlue.has(lastCodePoint);
|
|
378
|
+
}
|
|
379
|
+
function splitLeadingSpaceAndMarks(segment) {
|
|
380
|
+
if (segment.length < 2 || segment[0] !== ' ') return null;
|
|
381
|
+
const marks = segment.slice(1);
|
|
382
|
+
if (/^\p{M}+$/u.test(marks)) {
|
|
383
|
+
return {
|
|
384
|
+
space: ' ',
|
|
385
|
+
marks
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
return null;
|
|
389
|
+
}
|
|
390
|
+
export function endsWithClosingQuote(text) {
|
|
391
|
+
let end = text.length;
|
|
392
|
+
while(end > 0){
|
|
393
|
+
const start = previousCodePointStart(text, end);
|
|
394
|
+
const ch = text.slice(start, end);
|
|
395
|
+
if (closingQuoteChars.has(ch)) return true;
|
|
396
|
+
if (!leftStickyPunctuation.has(ch)) return false;
|
|
397
|
+
end = start;
|
|
398
|
+
}
|
|
399
|
+
return false;
|
|
400
|
+
}
|
|
401
|
+
function classifySegmentBreakChar(ch, whiteSpace) {
|
|
402
|
+
if (whiteSpace === 'pre-wrap') {
|
|
403
|
+
if (ch === ' ') return 'preserved-space';
|
|
404
|
+
if (ch === '\t') return 'tab';
|
|
405
|
+
if (ch === '\n') return 'hard-break';
|
|
406
|
+
}
|
|
407
|
+
if (ch === ' ') return 'space';
|
|
408
|
+
if (ch === '\u00A0' || ch === '\u202F' || ch === '\u2060' || ch === '\uFEFF') {
|
|
409
|
+
return 'glue';
|
|
410
|
+
}
|
|
411
|
+
if (ch === '\u200B') return 'zero-width-break';
|
|
412
|
+
if (ch === '\u00AD') return 'soft-hyphen';
|
|
413
|
+
return 'text';
|
|
414
|
+
}
|
|
415
|
+
const breakCharRe = /[\x20\t\n\xA0\xAD\u200B\u202F\u2060\uFEFF]/;
|
|
416
|
+
function joinTextParts(parts) {
|
|
417
|
+
return parts.length === 1 ? parts[0] : parts.join('');
|
|
418
|
+
}
|
|
419
|
+
function joinReversedPrefixParts(prefixParts, tail) {
|
|
420
|
+
const parts = [];
|
|
421
|
+
for(let i = prefixParts.length - 1; i >= 0; i--){
|
|
422
|
+
parts.push(prefixParts[i]);
|
|
423
|
+
}
|
|
424
|
+
parts.push(tail);
|
|
425
|
+
return joinTextParts(parts);
|
|
426
|
+
}
|
|
427
|
+
function splitSegmentByBreakKind(segment, isWordLike, start, whiteSpace) {
|
|
428
|
+
if (!breakCharRe.test(segment)) {
|
|
429
|
+
return [
|
|
430
|
+
{
|
|
431
|
+
text: segment,
|
|
432
|
+
isWordLike,
|
|
433
|
+
kind: 'text',
|
|
434
|
+
start
|
|
435
|
+
}
|
|
436
|
+
];
|
|
437
|
+
}
|
|
438
|
+
const pieces = [];
|
|
439
|
+
let currentKind = null;
|
|
440
|
+
let currentStart = 0;
|
|
441
|
+
let currentWordLike = false;
|
|
442
|
+
let offset = 0;
|
|
443
|
+
for (const ch of segment){
|
|
444
|
+
const kind = classifySegmentBreakChar(ch, whiteSpace);
|
|
445
|
+
const wordLike = kind === 'text' && isWordLike;
|
|
446
|
+
if (currentKind !== null && kind === currentKind && wordLike === currentWordLike) {
|
|
447
|
+
offset += ch.length;
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
if (currentKind !== null) {
|
|
451
|
+
pieces.push({
|
|
452
|
+
text: segment.slice(currentStart, offset),
|
|
453
|
+
isWordLike: currentWordLike,
|
|
454
|
+
kind: currentKind,
|
|
455
|
+
start: start + currentStart
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
currentKind = kind;
|
|
459
|
+
currentStart = offset;
|
|
460
|
+
currentWordLike = wordLike;
|
|
461
|
+
offset += ch.length;
|
|
462
|
+
}
|
|
463
|
+
if (currentKind !== null) {
|
|
464
|
+
pieces.push({
|
|
465
|
+
text: segment.slice(currentStart),
|
|
466
|
+
isWordLike: currentWordLike,
|
|
467
|
+
kind: currentKind,
|
|
468
|
+
start: start + currentStart
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
return pieces;
|
|
472
|
+
}
|
|
473
|
+
function isTextRunBoundary(kind) {
|
|
474
|
+
return kind === 'space' || kind === 'preserved-space' || kind === 'zero-width-break' || kind === 'hard-break';
|
|
475
|
+
}
|
|
476
|
+
const urlSchemeSegmentRe = /^[A-Za-z][A-Za-z0-9+.-]*:$/;
|
|
477
|
+
function isUrlLikeRunStart(segmentation, index) {
|
|
478
|
+
const text = segmentation.texts[index];
|
|
479
|
+
if (text.startsWith('www.')) return true;
|
|
480
|
+
return urlSchemeSegmentRe.test(text) && index + 1 < segmentation.len && segmentation.kinds[index + 1] === 'text' && segmentation.texts[index + 1] === '//';
|
|
481
|
+
}
|
|
482
|
+
function isUrlQueryBoundarySegment(text) {
|
|
483
|
+
return text.includes('?') && (text.includes('://') || text.startsWith('www.'));
|
|
484
|
+
}
|
|
485
|
+
function mergeUrlRuns(segmentation, normalized, profile) {
|
|
486
|
+
const texts = [];
|
|
487
|
+
const isWordLike = [];
|
|
488
|
+
const kinds = [];
|
|
489
|
+
const starts = [];
|
|
490
|
+
for(let i = 0; i < segmentation.len; i++){
|
|
491
|
+
const start = segmentation.starts[i];
|
|
492
|
+
let text = segmentation.texts[i];
|
|
493
|
+
let wordLike = segmentation.isWordLike[i];
|
|
494
|
+
const kind = segmentation.kinds[i];
|
|
495
|
+
let queryStartOverride = -1;
|
|
496
|
+
if (kind === 'text' && isUrlLikeRunStart(segmentation, i)) {
|
|
497
|
+
const urlParts = [
|
|
498
|
+
text
|
|
499
|
+
];
|
|
500
|
+
let j = i + 1;
|
|
501
|
+
while(j < segmentation.len && !isTextRunBoundary(segmentation.kinds[j]) && numericAffixBoundary(normalized, segmentation.starts[j], profile) !== false){
|
|
502
|
+
if (queryStartOverride < 0 && isUrlLikeRunStart(segmentation, j)) {
|
|
503
|
+
queryStartOverride = segmentation.starts[j];
|
|
504
|
+
}
|
|
505
|
+
const nextText = segmentation.texts[j];
|
|
506
|
+
urlParts.push(nextText);
|
|
507
|
+
wordLike = true;
|
|
508
|
+
j++;
|
|
509
|
+
if (nextText.includes('?')) break;
|
|
510
|
+
}
|
|
511
|
+
text = joinTextParts(urlParts);
|
|
512
|
+
i = j - 1;
|
|
513
|
+
}
|
|
514
|
+
texts.push(text);
|
|
515
|
+
isWordLike.push(wordLike);
|
|
516
|
+
kinds.push(kind);
|
|
517
|
+
starts.push(start);
|
|
518
|
+
if (!isUrlQueryBoundarySegment(text)) continue;
|
|
519
|
+
const nextIndex = i + 1;
|
|
520
|
+
if (nextIndex >= segmentation.len || isTextRunBoundary(segmentation.kinds[nextIndex])) {
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
const queryParts = [];
|
|
524
|
+
const queryStart = queryStartOverride < 0 ? segmentation.starts[nextIndex] : queryStartOverride;
|
|
525
|
+
let j = nextIndex;
|
|
526
|
+
while(j < segmentation.len && !isTextRunBoundary(segmentation.kinds[j]) && numericAffixBoundary(normalized, segmentation.starts[j], profile) !== false){
|
|
527
|
+
queryParts.push(segmentation.texts[j]);
|
|
528
|
+
j++;
|
|
529
|
+
}
|
|
530
|
+
if (queryParts.length > 0) {
|
|
531
|
+
texts.push(joinTextParts(queryParts));
|
|
532
|
+
isWordLike.push(true);
|
|
533
|
+
kinds.push('text');
|
|
534
|
+
starts.push(queryStart);
|
|
535
|
+
i = j - 1;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return {
|
|
539
|
+
len: texts.length,
|
|
540
|
+
texts,
|
|
541
|
+
isWordLike,
|
|
542
|
+
kinds,
|
|
543
|
+
starts
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
const numericJoinerChars = new Set([
|
|
547
|
+
':',
|
|
548
|
+
'-',
|
|
549
|
+
'/',
|
|
550
|
+
'×',
|
|
551
|
+
',',
|
|
552
|
+
'.',
|
|
553
|
+
'+',
|
|
554
|
+
'\u2013',
|
|
555
|
+
'\u2014'
|
|
556
|
+
]);
|
|
557
|
+
const wordInternalSymbolRe = /[\p{P}\p{S}\p{Co}]/u;
|
|
558
|
+
const emojiPresentationRe = /\p{Emoji_Presentation}/u;
|
|
559
|
+
const noSpaceWordBreakAfterChars = new Set([
|
|
560
|
+
'?',
|
|
561
|
+
'\u058A',
|
|
562
|
+
'-',
|
|
563
|
+
'\u2010',
|
|
564
|
+
'\u2012',
|
|
565
|
+
'\u2013',
|
|
566
|
+
'\u2014',
|
|
567
|
+
'\u2026',
|
|
568
|
+
'\u203C',
|
|
569
|
+
'\u203D',
|
|
570
|
+
'\u2049'
|
|
571
|
+
]);
|
|
572
|
+
function isAsciiWordInternalSymbolCode(code) {
|
|
573
|
+
return code >= 0x21 && code <= 0x2F && code !== 0x2D || code >= 0x3A && code <= 0x40 && code !== 0x3F || code >= 0x5B && code <= 0x60 || code >= 0x7B && code <= 0x7E;
|
|
574
|
+
}
|
|
575
|
+
function isNoSpaceWordInternalSymbol(ch) {
|
|
576
|
+
const code = ch.charCodeAt(0);
|
|
577
|
+
if (code < 0x80) return isAsciiWordInternalSymbolCode(code);
|
|
578
|
+
return !noSpaceWordBreakAfterChars.has(ch) && !emojiPresentationRe.test(ch) && wordInternalSymbolRe.test(ch);
|
|
579
|
+
}
|
|
580
|
+
function isNoSpaceWordInternalSymbolSegment(text) {
|
|
581
|
+
let sawSymbol = false;
|
|
582
|
+
for (const ch of text){
|
|
583
|
+
if (combiningMarkRe.test(ch)) continue;
|
|
584
|
+
if (!isNoSpaceWordInternalSymbol(ch)) return false;
|
|
585
|
+
sawSymbol = true;
|
|
586
|
+
}
|
|
587
|
+
return sawSymbol;
|
|
588
|
+
}
|
|
589
|
+
function endsWithNoSpaceWordJoiner(text) {
|
|
590
|
+
for(let end = text.length; end > 0;){
|
|
591
|
+
const start = previousCodePointStart(text, end);
|
|
592
|
+
const ch = text.slice(start, end);
|
|
593
|
+
if (combiningMarkRe.test(ch)) {
|
|
594
|
+
end = start;
|
|
595
|
+
continue;
|
|
596
|
+
}
|
|
597
|
+
return isNoSpaceWordInternalSymbol(ch) || isLineBreakNumericAffix(ch);
|
|
598
|
+
}
|
|
599
|
+
return false;
|
|
600
|
+
}
|
|
601
|
+
const asciiAlphabeticBoundaryRe = /[A-Za-z#&*<=>@^_`~]/;
|
|
602
|
+
function isAsciiBoundary(left, right) {
|
|
603
|
+
return left.charCodeAt(0) < 0x80 && right.charCodeAt(0) < 0x80;
|
|
604
|
+
}
|
|
605
|
+
function numericAffixBoundary(source, boundary, profile) {
|
|
606
|
+
if (!profile.geckoAsciiLineBreaks || boundary <= 0 || boundary >= source.length) return null;
|
|
607
|
+
const left = getLastSignificantCodePoint(source, boundary);
|
|
608
|
+
const right = String.fromCodePoint(source.codePointAt(boundary));
|
|
609
|
+
if (left === null || !isAsciiBoundary(left, right)) return null;
|
|
610
|
+
const leftAffix = isLineBreakNumericAffix(left);
|
|
611
|
+
const rightAffix = isLineBreakNumericAffix(right);
|
|
612
|
+
if (!leftAffix && !rightAffix) return null;
|
|
613
|
+
if (right === ')' || right === ']' || right === '}' || '!,.:;?/'.includes(right)) return true;
|
|
614
|
+
if ('([{'.includes(left) || left === '"' || left === "'" || right === '"' || right === "'") return true;
|
|
615
|
+
if (right === '|' || right === '-') return true;
|
|
616
|
+
if (leftAffix && asciiAlphabeticBoundaryRe.test(right) || asciiAlphabeticBoundaryRe.test(left) && rightAffix) return true;
|
|
617
|
+
if (rightAffix && (')]}'.includes(left) || /[0-9]/.test(left))) return true;
|
|
618
|
+
if (leftAffix && ('([{'.includes(right) || /[0-9]/.test(right))) return true;
|
|
619
|
+
return false;
|
|
620
|
+
}
|
|
621
|
+
function openingPunctuationJoinsPrevious(left, right, profile, leftEnd = left.length) {
|
|
622
|
+
const first = right[0];
|
|
623
|
+
if (first === undefined || !'([{'.includes(first)) return null;
|
|
624
|
+
const last = getLastSignificantCodePoint(left, leftEnd);
|
|
625
|
+
if (last === null) return null;
|
|
626
|
+
if (profile.geckoAsciiLineBreaks && isAsciiBoundary(last, first)) {
|
|
627
|
+
return asciiAlphabeticBoundaryRe.test(last) || decimalDigitRe.test(last) || '"\'([{'.includes(last) || isLineBreakNumericAffix(last);
|
|
628
|
+
}
|
|
629
|
+
if (last.charCodeAt(0) >= 0x80) {
|
|
630
|
+
return !isCJK(last) && /[\p{L}\p{N}]/u.test(last) ? true : null;
|
|
631
|
+
}
|
|
632
|
+
return /[A-Za-z0-9]/.test(last) || "$'(/<@[^_`{".includes(last);
|
|
633
|
+
}
|
|
634
|
+
function canJoinNoSpaceWordBoundary(leftText, leftWordLike, rightText, rightWordLike, profile) {
|
|
635
|
+
if (rightText[0] === '-' && isCJK(leftText)) return true;
|
|
636
|
+
if (isCJK(leftText) || isCJK(rightText)) return false;
|
|
637
|
+
const openingJoin = openingPunctuationJoinsPrevious(leftText, rightText, profile);
|
|
638
|
+
if (openingJoin !== null) return openingJoin;
|
|
639
|
+
const leftSymbol = !leftWordLike && isNoSpaceWordInternalSymbolSegment(leftText);
|
|
640
|
+
const rightSymbol = !rightWordLike && isNoSpaceWordInternalSymbolSegment(rightText);
|
|
641
|
+
const leftAffix = endsWithLineBreakNumericAffix(leftText);
|
|
642
|
+
const leftEndsJoiner = (leftWordLike || leftAffix) && endsWithNoSpaceWordJoiner(leftText);
|
|
643
|
+
if (!leftSymbol && !rightSymbol && !leftEndsJoiner) return false;
|
|
644
|
+
return (leftWordLike || leftSymbol || leftAffix) && (rightWordLike || rightSymbol);
|
|
645
|
+
}
|
|
646
|
+
function segmentContainsDecimalDigit(text) {
|
|
647
|
+
for (const ch of text){
|
|
648
|
+
if (decimalDigitRe.test(ch)) return true;
|
|
649
|
+
}
|
|
650
|
+
return false;
|
|
651
|
+
}
|
|
652
|
+
export function isNumericRunSegment(text) {
|
|
653
|
+
if (text.length === 0) return false;
|
|
654
|
+
for (const ch of text){
|
|
655
|
+
if (decimalDigitRe.test(ch) || numericJoinerChars.has(ch)) continue;
|
|
656
|
+
return false;
|
|
657
|
+
}
|
|
658
|
+
return true;
|
|
659
|
+
}
|
|
660
|
+
function mergeNumericRuns(segmentation, normalized, profile) {
|
|
661
|
+
const texts = [];
|
|
662
|
+
const isWordLike = [];
|
|
663
|
+
const kinds = [];
|
|
664
|
+
const starts = [];
|
|
665
|
+
function pushNumericRun(text, start) {
|
|
666
|
+
if (text.includes('-')) {
|
|
667
|
+
const parts = text.split('-');
|
|
668
|
+
let shouldSplit = parts.length > 1;
|
|
669
|
+
for(let i = 0; i < parts.length; i++){
|
|
670
|
+
const part = parts[i];
|
|
671
|
+
if (!shouldSplit) break;
|
|
672
|
+
if (part.length === 0 || !segmentContainsDecimalDigit(part) || !isNumericRunSegment(part)) {
|
|
673
|
+
shouldSplit = false;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
if (shouldSplit) {
|
|
677
|
+
let offset = 0;
|
|
678
|
+
for(let i = 0; i < parts.length; i++){
|
|
679
|
+
const part = parts[i];
|
|
680
|
+
const splitText = i < parts.length - 1 ? `${part}-` : part;
|
|
681
|
+
texts.push(splitText);
|
|
682
|
+
isWordLike.push(true);
|
|
683
|
+
kinds.push('text');
|
|
684
|
+
starts.push(start + offset);
|
|
685
|
+
offset += splitText.length;
|
|
686
|
+
}
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
texts.push(text);
|
|
691
|
+
isWordLike.push(true);
|
|
692
|
+
kinds.push('text');
|
|
693
|
+
starts.push(start);
|
|
694
|
+
}
|
|
695
|
+
for(let i = 0; i < segmentation.len; i++){
|
|
696
|
+
const text = segmentation.texts[i];
|
|
697
|
+
const kind = segmentation.kinds[i];
|
|
698
|
+
if (kind === 'text' && isNumericRunSegment(text) && segmentContainsDecimalDigit(text)) {
|
|
699
|
+
const mergedParts = [
|
|
700
|
+
text
|
|
701
|
+
];
|
|
702
|
+
let j = i + 1;
|
|
703
|
+
while(j < segmentation.len && segmentation.kinds[j] === 'text' && isNumericRunSegment(segmentation.texts[j]) && numericAffixBoundary(normalized, segmentation.starts[j], profile) !== false){
|
|
704
|
+
mergedParts.push(segmentation.texts[j]);
|
|
705
|
+
j++;
|
|
706
|
+
}
|
|
707
|
+
pushNumericRun(joinTextParts(mergedParts), segmentation.starts[i]);
|
|
708
|
+
i = j - 1;
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
711
|
+
texts.push(text);
|
|
712
|
+
isWordLike.push(segmentation.isWordLike[i]);
|
|
713
|
+
kinds.push(kind);
|
|
714
|
+
starts.push(segmentation.starts[i]);
|
|
715
|
+
}
|
|
716
|
+
return {
|
|
717
|
+
len: texts.length,
|
|
718
|
+
texts,
|
|
719
|
+
isWordLike,
|
|
720
|
+
kinds,
|
|
721
|
+
starts
|
|
722
|
+
};
|
|
723
|
+
}
|
|
724
|
+
function mergeNoSpaceWordChains(segmentation, normalized, profile) {
|
|
725
|
+
const texts = [];
|
|
726
|
+
const isWordLike = [];
|
|
727
|
+
const kinds = [];
|
|
728
|
+
const starts = [];
|
|
729
|
+
let i = 0;
|
|
730
|
+
while(i < segmentation.len){
|
|
731
|
+
const text = segmentation.texts[i];
|
|
732
|
+
const kind = segmentation.kinds[i];
|
|
733
|
+
const wordLike = segmentation.isWordLike[i];
|
|
734
|
+
if (kind === 'text') {
|
|
735
|
+
const mergedParts = [
|
|
736
|
+
text
|
|
737
|
+
];
|
|
738
|
+
let j = i + 1;
|
|
739
|
+
let mergedWordLike = wordLike;
|
|
740
|
+
while(j < segmentation.len && segmentation.kinds[j] === 'text' && (numericAffixBoundary(normalized, segmentation.starts[j], profile) ?? canJoinNoSpaceWordBoundary(segmentation.texts[j - 1], segmentation.isWordLike[j - 1], segmentation.texts[j], segmentation.isWordLike[j], profile))){
|
|
741
|
+
const nextText = segmentation.texts[j];
|
|
742
|
+
mergedParts.push(nextText);
|
|
743
|
+
mergedWordLike = mergedWordLike || segmentation.isWordLike[j];
|
|
744
|
+
j++;
|
|
745
|
+
}
|
|
746
|
+
if (j > i + 1) {
|
|
747
|
+
texts.push(joinTextParts(mergedParts));
|
|
748
|
+
isWordLike.push(mergedWordLike);
|
|
749
|
+
kinds.push('text');
|
|
750
|
+
starts.push(segmentation.starts[i]);
|
|
751
|
+
i = j;
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
texts.push(text);
|
|
756
|
+
isWordLike.push(wordLike);
|
|
757
|
+
kinds.push(kind);
|
|
758
|
+
starts.push(segmentation.starts[i]);
|
|
759
|
+
i++;
|
|
760
|
+
}
|
|
761
|
+
return {
|
|
762
|
+
len: texts.length,
|
|
763
|
+
texts,
|
|
764
|
+
isWordLike,
|
|
765
|
+
kinds,
|
|
766
|
+
starts
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
function mergeGlueConnectedTextRuns(segmentation) {
|
|
770
|
+
const texts = [];
|
|
771
|
+
const isWordLike = [];
|
|
772
|
+
const kinds = [];
|
|
773
|
+
const starts = [];
|
|
774
|
+
let read = 0;
|
|
775
|
+
while(read < segmentation.len){
|
|
776
|
+
const textParts = [
|
|
777
|
+
segmentation.texts[read]
|
|
778
|
+
];
|
|
779
|
+
let wordLike = segmentation.isWordLike[read];
|
|
780
|
+
let kind = segmentation.kinds[read];
|
|
781
|
+
let start = segmentation.starts[read];
|
|
782
|
+
if (kind === 'glue') {
|
|
783
|
+
const glueParts = [
|
|
784
|
+
textParts[0]
|
|
785
|
+
];
|
|
786
|
+
const glueStart = start;
|
|
787
|
+
read++;
|
|
788
|
+
while(read < segmentation.len && segmentation.kinds[read] === 'glue'){
|
|
789
|
+
glueParts.push(segmentation.texts[read]);
|
|
790
|
+
read++;
|
|
791
|
+
}
|
|
792
|
+
const glueText = joinTextParts(glueParts);
|
|
793
|
+
if (read < segmentation.len && segmentation.kinds[read] === 'text') {
|
|
794
|
+
textParts[0] = glueText;
|
|
795
|
+
textParts.push(segmentation.texts[read]);
|
|
796
|
+
wordLike = segmentation.isWordLike[read];
|
|
797
|
+
kind = 'text';
|
|
798
|
+
start = glueStart;
|
|
799
|
+
read++;
|
|
800
|
+
} else {
|
|
801
|
+
texts.push(glueText);
|
|
802
|
+
isWordLike.push(false);
|
|
803
|
+
kinds.push('glue');
|
|
804
|
+
starts.push(glueStart);
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
} else {
|
|
808
|
+
read++;
|
|
809
|
+
}
|
|
810
|
+
if (kind === 'text') {
|
|
811
|
+
while(read < segmentation.len && segmentation.kinds[read] === 'glue'){
|
|
812
|
+
const glueParts = [];
|
|
813
|
+
while(read < segmentation.len && segmentation.kinds[read] === 'glue'){
|
|
814
|
+
glueParts.push(segmentation.texts[read]);
|
|
815
|
+
read++;
|
|
816
|
+
}
|
|
817
|
+
const glueText = joinTextParts(glueParts);
|
|
818
|
+
if (read < segmentation.len && segmentation.kinds[read] === 'text') {
|
|
819
|
+
textParts.push(glueText, segmentation.texts[read]);
|
|
820
|
+
wordLike = wordLike || segmentation.isWordLike[read];
|
|
821
|
+
read++;
|
|
822
|
+
continue;
|
|
823
|
+
}
|
|
824
|
+
textParts.push(glueText);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
texts.push(joinTextParts(textParts));
|
|
828
|
+
isWordLike.push(wordLike);
|
|
829
|
+
kinds.push(kind);
|
|
830
|
+
starts.push(start);
|
|
831
|
+
}
|
|
832
|
+
return {
|
|
833
|
+
len: texts.length,
|
|
834
|
+
texts,
|
|
835
|
+
isWordLike,
|
|
836
|
+
kinds,
|
|
837
|
+
starts
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
function carryTrailingForwardStickyAcrossCJKBoundary(segmentation) {
|
|
841
|
+
const { texts, kinds, starts } = segmentation;
|
|
842
|
+
for(let i = 0; i < texts.length - 1; i++){
|
|
843
|
+
if (kinds[i] !== 'text' || kinds[i + 1] !== 'text') continue;
|
|
844
|
+
if (!isCJK(texts[i]) || !isCJK(texts[i + 1])) continue;
|
|
845
|
+
const split = splitTrailingForwardStickyCluster(texts[i]);
|
|
846
|
+
if (split === null) continue;
|
|
847
|
+
texts[i] = split.head;
|
|
848
|
+
texts[i + 1] = split.tail + texts[i + 1];
|
|
849
|
+
starts[i + 1] = starts[i] + split.head.length;
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
function buildMergedSegmentation(normalized, profile, whiteSpace) {
|
|
853
|
+
const wordSegmenter = getSharedWordSegmenter();
|
|
854
|
+
let mergedLen = 0;
|
|
855
|
+
const mergedTexts = [];
|
|
856
|
+
const mergedWordLike = [];
|
|
857
|
+
const mergedKinds = [];
|
|
858
|
+
const mergedStarts = [];
|
|
859
|
+
let hasTail = false;
|
|
860
|
+
let tailStart = 0;
|
|
861
|
+
let tailEnd = 0;
|
|
862
|
+
let tailWordLike = false;
|
|
863
|
+
let tailKind = 'text';
|
|
864
|
+
let tailSingleCharRunChar = null;
|
|
865
|
+
let tailContainsCJK = false;
|
|
866
|
+
let tailContainsArabicScript = false;
|
|
867
|
+
let tailEndsWithClosingQuote = false;
|
|
868
|
+
let tailEndsWithMyanmarMedialGlue = false;
|
|
869
|
+
let tailHasArabicNoSpacePunctuation = false;
|
|
870
|
+
for (const s of wordSegmenter.segment(normalized)){
|
|
871
|
+
for (const piece of splitSegmentByBreakKind(s.segment, s.isWordLike ?? false, s.index, whiteSpace)){
|
|
872
|
+
const isText = piece.kind === 'text';
|
|
873
|
+
const repeatableSingleCharRunChar = getRepeatableSingleCharRunChar(piece.text, piece.isWordLike, piece.kind);
|
|
874
|
+
const pieceContainsCJK = isCJK(piece.text);
|
|
875
|
+
const pieceContainsArabicScript = containsArabicScript(piece.text);
|
|
876
|
+
const pieceLastCodePoint = getLastCodePoint(piece.text);
|
|
877
|
+
const pieceEndsWithClosingQuote = endsWithClosingQuote(piece.text);
|
|
878
|
+
const pieceEndsWithMyanmarMedialGlue = endsWithMyanmarMedialGlue(piece.text);
|
|
879
|
+
const pieceEnd = piece.start + piece.text.length;
|
|
880
|
+
const boundaryJoin = numericAffixBoundary(normalized, piece.start, profile) ?? (tailContainsCJK || pieceContainsCJK ? null : openingPunctuationJoinsPrevious(normalized, piece.text, profile, piece.start));
|
|
881
|
+
let appendToTail = false;
|
|
882
|
+
if (profile.carryCJKAfterClosingQuote && isText && hasTail && tailKind === 'text' && pieceContainsCJK && tailContainsCJK && tailEndsWithClosingQuote) {
|
|
883
|
+
appendToTail = true;
|
|
884
|
+
} else if (isText && hasTail && tailKind === 'text' && isCJKLineStartProhibitedSegment(piece.text) && tailContainsCJK) {
|
|
885
|
+
appendToTail = true;
|
|
886
|
+
} else if (isText && hasTail && tailKind === 'text' && tailEndsWithMyanmarMedialGlue) {
|
|
887
|
+
appendToTail = true;
|
|
888
|
+
} else if (isText && hasTail && tailKind === 'text' && piece.isWordLike && pieceContainsArabicScript && tailHasArabicNoSpacePunctuation) {
|
|
889
|
+
appendToTail = true;
|
|
890
|
+
} else if (repeatableSingleCharRunChar !== null && hasTail && tailKind === 'text' && tailSingleCharRunChar === repeatableSingleCharRunChar && boundaryJoin !== false) {
|
|
891
|
+
tailEnd = pieceEnd;
|
|
892
|
+
continue;
|
|
893
|
+
} else if (isText && !piece.isWordLike && hasTail && tailKind === 'text' && !tailContainsCJK && (isLeftStickyPunctuationSegment(piece.text) || piece.text === '-' && tailWordLike)) {
|
|
894
|
+
appendToTail = true;
|
|
895
|
+
}
|
|
896
|
+
if (isText && hasTail && tailKind === 'text' && boundaryJoin !== null) appendToTail = boundaryJoin;
|
|
897
|
+
if (appendToTail) {
|
|
898
|
+
tailEnd = pieceEnd;
|
|
899
|
+
tailWordLike = tailWordLike || piece.isWordLike;
|
|
900
|
+
tailSingleCharRunChar = null;
|
|
901
|
+
tailContainsCJK = tailContainsCJK || pieceContainsCJK;
|
|
902
|
+
tailContainsArabicScript = tailContainsArabicScript || pieceContainsArabicScript;
|
|
903
|
+
tailEndsWithClosingQuote = pieceEndsWithClosingQuote;
|
|
904
|
+
tailEndsWithMyanmarMedialGlue = pieceEndsWithMyanmarMedialGlue;
|
|
905
|
+
tailHasArabicNoSpacePunctuation = hasArabicNoSpacePunctuation(tailContainsArabicScript, pieceLastCodePoint);
|
|
906
|
+
} else {
|
|
907
|
+
if (hasTail) {
|
|
908
|
+
mergedTexts[mergedLen] = normalized.slice(tailStart, tailEnd);
|
|
909
|
+
mergedWordLike[mergedLen] = tailWordLike;
|
|
910
|
+
mergedKinds[mergedLen] = tailKind;
|
|
911
|
+
mergedStarts[mergedLen] = tailStart;
|
|
912
|
+
mergedLen++;
|
|
913
|
+
}
|
|
914
|
+
hasTail = true;
|
|
915
|
+
tailStart = piece.start;
|
|
916
|
+
tailEnd = pieceEnd;
|
|
917
|
+
tailWordLike = piece.isWordLike;
|
|
918
|
+
tailKind = piece.kind;
|
|
919
|
+
tailSingleCharRunChar = repeatableSingleCharRunChar;
|
|
920
|
+
tailContainsCJK = pieceContainsCJK;
|
|
921
|
+
tailContainsArabicScript = pieceContainsArabicScript;
|
|
922
|
+
tailEndsWithClosingQuote = pieceEndsWithClosingQuote;
|
|
923
|
+
tailEndsWithMyanmarMedialGlue = pieceEndsWithMyanmarMedialGlue;
|
|
924
|
+
tailHasArabicNoSpacePunctuation = hasArabicNoSpacePunctuation(pieceContainsArabicScript, pieceLastCodePoint);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
if (hasTail) {
|
|
929
|
+
mergedTexts[mergedLen] = normalized.slice(tailStart, tailEnd);
|
|
930
|
+
mergedWordLike[mergedLen] = tailWordLike;
|
|
931
|
+
mergedKinds[mergedLen] = tailKind;
|
|
932
|
+
mergedStarts[mergedLen] = tailStart;
|
|
933
|
+
mergedLen++;
|
|
934
|
+
}
|
|
935
|
+
for(let i = 1; i < mergedLen; i++){
|
|
936
|
+
if (mergedKinds[i] === 'text' && !mergedWordLike[i] && isPunctuationGlueCluster(mergedTexts[i]) && mergedKinds[i - 1] === 'text' && !isCJK(mergedTexts[i - 1]) && (numericAffixBoundary(normalized, mergedStarts[i], profile) ?? openingPunctuationJoinsPrevious(normalized, mergedTexts[i], profile, mergedStarts[i])) !== false) {
|
|
937
|
+
mergedTexts[i - 1] += mergedTexts[i];
|
|
938
|
+
mergedWordLike[i - 1] = mergedWordLike[i - 1] || mergedWordLike[i];
|
|
939
|
+
mergedTexts[i] = '';
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
let nextLiveIndex = -1;
|
|
943
|
+
let forwardStickyPrefixParts = null;
|
|
944
|
+
for(let i = mergedLen - 1; i >= 0; i--){
|
|
945
|
+
const text = mergedTexts[i];
|
|
946
|
+
if (text.length === 0) continue;
|
|
947
|
+
const nextText = forwardStickyPrefixParts?.at(-1) ?? (nextLiveIndex >= 0 ? mergedTexts[nextLiveIndex] : null);
|
|
948
|
+
if (mergedKinds[i] === 'text' && !mergedWordLike[i] && nextText !== null && mergedKinds[nextLiveIndex] === 'text' && ((numericAffixBoundary(normalized, mergedStarts[i] + text.length, profile) ?? (isForwardStickyClusterSegment(text) && openingPunctuationJoinsPrevious(text, nextText, profile) !== false)) || text === '-' && startsWithDecimalDigit(nextText))) {
|
|
949
|
+
if (forwardStickyPrefixParts === null) forwardStickyPrefixParts = [];
|
|
950
|
+
forwardStickyPrefixParts.push(text);
|
|
951
|
+
mergedStarts[nextLiveIndex] = mergedStarts[i];
|
|
952
|
+
mergedTexts[i] = '';
|
|
953
|
+
continue;
|
|
954
|
+
}
|
|
955
|
+
if (forwardStickyPrefixParts !== null) {
|
|
956
|
+
mergedTexts[nextLiveIndex] = joinReversedPrefixParts(forwardStickyPrefixParts, mergedTexts[nextLiveIndex]);
|
|
957
|
+
forwardStickyPrefixParts = null;
|
|
958
|
+
}
|
|
959
|
+
nextLiveIndex = i;
|
|
960
|
+
}
|
|
961
|
+
if (forwardStickyPrefixParts !== null) {
|
|
962
|
+
mergedTexts[nextLiveIndex] = joinReversedPrefixParts(forwardStickyPrefixParts, mergedTexts[nextLiveIndex]);
|
|
963
|
+
}
|
|
964
|
+
let compactLen = 0;
|
|
965
|
+
for(let read = 0; read < mergedLen; read++){
|
|
966
|
+
const text = mergedTexts[read];
|
|
967
|
+
if (text.length === 0) continue;
|
|
968
|
+
if (compactLen !== read) {
|
|
969
|
+
mergedTexts[compactLen] = text;
|
|
970
|
+
mergedWordLike[compactLen] = mergedWordLike[read];
|
|
971
|
+
mergedKinds[compactLen] = mergedKinds[read];
|
|
972
|
+
mergedStarts[compactLen] = mergedStarts[read];
|
|
973
|
+
}
|
|
974
|
+
compactLen++;
|
|
975
|
+
}
|
|
976
|
+
mergedTexts.length = compactLen;
|
|
977
|
+
mergedWordLike.length = compactLen;
|
|
978
|
+
mergedKinds.length = compactLen;
|
|
979
|
+
mergedStarts.length = compactLen;
|
|
980
|
+
const compacted = mergeGlueConnectedTextRuns({
|
|
981
|
+
len: compactLen,
|
|
982
|
+
texts: mergedTexts,
|
|
983
|
+
isWordLike: mergedWordLike,
|
|
984
|
+
kinds: mergedKinds,
|
|
985
|
+
starts: mergedStarts
|
|
986
|
+
});
|
|
987
|
+
const mergedRuns = mergeNoSpaceWordChains(mergeNumericRuns(mergeUrlRuns(compacted, normalized, profile), normalized, profile), normalized, profile);
|
|
988
|
+
carryTrailingForwardStickyAcrossCJKBoundary(mergedRuns);
|
|
989
|
+
for(let i = 0; i < mergedRuns.len - 1; i++){
|
|
990
|
+
const split = splitLeadingSpaceAndMarks(mergedRuns.texts[i]);
|
|
991
|
+
if (split === null) continue;
|
|
992
|
+
if (mergedRuns.kinds[i] !== 'space' && mergedRuns.kinds[i] !== 'preserved-space' || mergedRuns.kinds[i + 1] !== 'text' || !containsArabicScript(mergedRuns.texts[i + 1])) {
|
|
993
|
+
continue;
|
|
994
|
+
}
|
|
995
|
+
mergedRuns.texts[i] = split.space;
|
|
996
|
+
mergedRuns.isWordLike[i] = false;
|
|
997
|
+
mergedRuns.kinds[i] = mergedRuns.kinds[i] === 'preserved-space' ? 'preserved-space' : 'space';
|
|
998
|
+
mergedRuns.texts[i + 1] = split.marks + mergedRuns.texts[i + 1];
|
|
999
|
+
mergedRuns.starts[i + 1] = mergedRuns.starts[i] + split.space.length;
|
|
1000
|
+
}
|
|
1001
|
+
return mergedRuns;
|
|
1002
|
+
}
|
|
1003
|
+
function mergeKeepAllTextSegments(normalized, segmentation, profile) {
|
|
1004
|
+
if (segmentation.len <= 1) return segmentation;
|
|
1005
|
+
const texts = [];
|
|
1006
|
+
const isWordLike = [];
|
|
1007
|
+
const kinds = [];
|
|
1008
|
+
const starts = [];
|
|
1009
|
+
let groupStart = -1;
|
|
1010
|
+
let groupContainsCJK = false;
|
|
1011
|
+
function pushOriginalText(index) {
|
|
1012
|
+
texts.push(segmentation.texts[index]);
|
|
1013
|
+
isWordLike.push(segmentation.isWordLike[index]);
|
|
1014
|
+
kinds.push('text');
|
|
1015
|
+
starts.push(segmentation.starts[index]);
|
|
1016
|
+
}
|
|
1017
|
+
function pushMergedText(start, end) {
|
|
1018
|
+
let wordLike = false;
|
|
1019
|
+
for(let i = start; i < end; i++){
|
|
1020
|
+
wordLike = wordLike || segmentation.isWordLike[i];
|
|
1021
|
+
}
|
|
1022
|
+
const sourceStart = segmentation.starts[start];
|
|
1023
|
+
const sourceEnd = end < segmentation.len ? segmentation.starts[end] : normalized.length;
|
|
1024
|
+
texts.push(normalized.slice(sourceStart, sourceEnd));
|
|
1025
|
+
isWordLike.push(wordLike);
|
|
1026
|
+
kinds.push('text');
|
|
1027
|
+
starts.push(sourceStart);
|
|
1028
|
+
}
|
|
1029
|
+
function flushGroup(end) {
|
|
1030
|
+
if (groupStart < 0) return;
|
|
1031
|
+
if (groupContainsCJK) {
|
|
1032
|
+
if (groupStart + 1 === end) {
|
|
1033
|
+
pushOriginalText(groupStart);
|
|
1034
|
+
} else {
|
|
1035
|
+
pushMergedText(groupStart, end);
|
|
1036
|
+
}
|
|
1037
|
+
} else {
|
|
1038
|
+
for(let i = groupStart; i < end; i++)pushOriginalText(i);
|
|
1039
|
+
}
|
|
1040
|
+
groupStart = -1;
|
|
1041
|
+
groupContainsCJK = false;
|
|
1042
|
+
}
|
|
1043
|
+
for(let i = 0; i < segmentation.len; i++){
|
|
1044
|
+
const text = segmentation.texts[i];
|
|
1045
|
+
const kind = segmentation.kinds[i];
|
|
1046
|
+
if (kind === 'text') {
|
|
1047
|
+
if (groupStart >= 0 && (!canContinueKeepAllTextRun(segmentation.texts[i - 1], profile.breakKeepAllAfterPunctuation) || numericAffixBoundary(normalized, segmentation.starts[i], profile) === false)) {
|
|
1048
|
+
flushGroup(i);
|
|
1049
|
+
}
|
|
1050
|
+
if (groupStart < 0) groupStart = i;
|
|
1051
|
+
groupContainsCJK = groupContainsCJK || isCJK(text);
|
|
1052
|
+
continue;
|
|
1053
|
+
}
|
|
1054
|
+
flushGroup(i);
|
|
1055
|
+
texts.push(text);
|
|
1056
|
+
isWordLike.push(segmentation.isWordLike[i]);
|
|
1057
|
+
kinds.push(kind);
|
|
1058
|
+
starts.push(segmentation.starts[i]);
|
|
1059
|
+
}
|
|
1060
|
+
flushGroup(segmentation.len);
|
|
1061
|
+
return {
|
|
1062
|
+
len: texts.length,
|
|
1063
|
+
texts,
|
|
1064
|
+
isWordLike,
|
|
1065
|
+
kinds,
|
|
1066
|
+
starts
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
function isNumericHyphen(text, index) {
|
|
1070
|
+
const next = text.charCodeAt(index + 1);
|
|
1071
|
+
if (next < 0x80) {
|
|
1072
|
+
if (next < 0x30 || next > 0x39) return false;
|
|
1073
|
+
} else {
|
|
1074
|
+
const codePoint = text.codePointAt(index + 1);
|
|
1075
|
+
if (codePoint === undefined || !decimalDigitRe.test(String.fromCodePoint(codePoint))) return false;
|
|
1076
|
+
}
|
|
1077
|
+
for(let end = index; end > 0;){
|
|
1078
|
+
const start = previousCodePointStart(text, end);
|
|
1079
|
+
const previous = text.slice(start, end);
|
|
1080
|
+
if (!combiningMarkRe.test(previous)) return isCJK(previous) || !/[\p{L}\p{N}]/u.test(previous);
|
|
1081
|
+
end = start;
|
|
1082
|
+
}
|
|
1083
|
+
return true;
|
|
1084
|
+
}
|
|
1085
|
+
function buildBaseCjkUnits(segText, profile) {
|
|
1086
|
+
const units = [];
|
|
1087
|
+
let unitStart = 0;
|
|
1088
|
+
let unitEnd = 0;
|
|
1089
|
+
let unitContainsCJK = false;
|
|
1090
|
+
let unitEndsWithClosingQuote = false;
|
|
1091
|
+
let unitIsSingleKinsokuEnd = false;
|
|
1092
|
+
let unitHasHyphen = false;
|
|
1093
|
+
let unitHasNumericHyphen = false;
|
|
1094
|
+
function pushUnit() {
|
|
1095
|
+
if (unitEnd === unitStart) return;
|
|
1096
|
+
units.push({
|
|
1097
|
+
text: segText.slice(unitStart, unitEnd),
|
|
1098
|
+
start: unitStart,
|
|
1099
|
+
overflow: unitContainsCJK ? unitHasHyphen ? 'grapheme' : 'none' : 'word-like'
|
|
1100
|
+
});
|
|
1101
|
+
unitStart = unitEnd;
|
|
1102
|
+
unitContainsCJK = false;
|
|
1103
|
+
unitEndsWithClosingQuote = false;
|
|
1104
|
+
unitIsSingleKinsokuEnd = false;
|
|
1105
|
+
unitHasHyphen = false;
|
|
1106
|
+
unitHasNumericHyphen = false;
|
|
1107
|
+
}
|
|
1108
|
+
function startUnit(grapheme, start, graphemeContainsCJK) {
|
|
1109
|
+
unitStart = start;
|
|
1110
|
+
unitEnd = start + grapheme.length;
|
|
1111
|
+
unitContainsCJK = graphemeContainsCJK;
|
|
1112
|
+
unitHasHyphen = grapheme === '-';
|
|
1113
|
+
unitEndsWithClosingQuote = endsWithClosingQuote(grapheme);
|
|
1114
|
+
unitIsSingleKinsokuEnd = kinsokuEnd.has(grapheme);
|
|
1115
|
+
}
|
|
1116
|
+
function appendToUnit(grapheme, graphemeContainsCJK) {
|
|
1117
|
+
unitEnd += grapheme.length;
|
|
1118
|
+
unitContainsCJK = unitContainsCJK || graphemeContainsCJK;
|
|
1119
|
+
unitHasHyphen = unitHasHyphen || grapheme === '-';
|
|
1120
|
+
const graphemeEndsWithClosingQuote = endsWithClosingQuote(grapheme);
|
|
1121
|
+
if (grapheme.length === 1 && leftStickyPunctuation.has(grapheme)) {
|
|
1122
|
+
unitEndsWithClosingQuote = unitEndsWithClosingQuote || graphemeEndsWithClosingQuote;
|
|
1123
|
+
} else {
|
|
1124
|
+
unitEndsWithClosingQuote = graphemeEndsWithClosingQuote;
|
|
1125
|
+
}
|
|
1126
|
+
unitIsSingleKinsokuEnd = false;
|
|
1127
|
+
}
|
|
1128
|
+
for (const gs of getSharedGraphemeSegmenter().segment(segText)){
|
|
1129
|
+
const grapheme = gs.segment;
|
|
1130
|
+
const graphemeContainsCJK = isCJK(grapheme);
|
|
1131
|
+
if (unitEnd === unitStart) {
|
|
1132
|
+
startUnit(grapheme, gs.index, graphemeContainsCJK);
|
|
1133
|
+
continue;
|
|
1134
|
+
}
|
|
1135
|
+
const attachHyphen = grapheme === '-' && unitContainsCJK;
|
|
1136
|
+
if (attachHyphen && isNumericHyphen(segText, gs.index)) unitHasNumericHyphen = true;
|
|
1137
|
+
if (unitIsSingleKinsokuEnd || kinsokuStart.has(grapheme) || leftStickyPunctuation.has(grapheme) || attachHyphen || unitHasNumericHyphen && !graphemeContainsCJK || profile.carryCJKAfterClosingQuote && graphemeContainsCJK && unitEndsWithClosingQuote) {
|
|
1138
|
+
appendToUnit(grapheme, graphemeContainsCJK);
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
if (!unitContainsCJK && !graphemeContainsCJK) {
|
|
1142
|
+
appendToUnit(grapheme, graphemeContainsCJK);
|
|
1143
|
+
continue;
|
|
1144
|
+
}
|
|
1145
|
+
pushUnit();
|
|
1146
|
+
startUnit(grapheme, gs.index, graphemeContainsCJK);
|
|
1147
|
+
}
|
|
1148
|
+
pushUnit();
|
|
1149
|
+
return units;
|
|
1150
|
+
}
|
|
1151
|
+
function mergeKeepAllTextUnits(segText, units, profile) {
|
|
1152
|
+
if (units.length <= 1) return units;
|
|
1153
|
+
const merged = [];
|
|
1154
|
+
let groupStart = -1;
|
|
1155
|
+
let groupContainsCJK = false;
|
|
1156
|
+
function pushMergedUnit(start, end) {
|
|
1157
|
+
const sourceStart = units[start].start;
|
|
1158
|
+
const sourceEnd = end < units.length ? units[end].start : segText.length;
|
|
1159
|
+
merged.push({
|
|
1160
|
+
text: segText.slice(sourceStart, sourceEnd),
|
|
1161
|
+
start: sourceStart,
|
|
1162
|
+
overflow: 'word-like'
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
function flushGroup(end) {
|
|
1166
|
+
if (groupStart < 0) return;
|
|
1167
|
+
if (groupContainsCJK) {
|
|
1168
|
+
if (groupStart + 1 === end) {
|
|
1169
|
+
merged.push(units[groupStart]);
|
|
1170
|
+
} else {
|
|
1171
|
+
pushMergedUnit(groupStart, end);
|
|
1172
|
+
}
|
|
1173
|
+
} else {
|
|
1174
|
+
for(let i = groupStart; i < end; i++)merged.push(units[i]);
|
|
1175
|
+
}
|
|
1176
|
+
groupStart = -1;
|
|
1177
|
+
groupContainsCJK = false;
|
|
1178
|
+
}
|
|
1179
|
+
for(let i = 0; i < units.length; i++){
|
|
1180
|
+
const unit = units[i];
|
|
1181
|
+
if (groupStart >= 0 && (!canContinueKeepAllTextRun(units[i - 1].text, profile.breakKeepAllAfterPunctuation) || numericAffixBoundary(segText, unit.start, profile) === false)) {
|
|
1182
|
+
flushGroup(i);
|
|
1183
|
+
}
|
|
1184
|
+
if (groupStart < 0) groupStart = i;
|
|
1185
|
+
groupContainsCJK = groupContainsCJK || isCJK(unit.text);
|
|
1186
|
+
}
|
|
1187
|
+
flushGroup(units.length);
|
|
1188
|
+
return merged;
|
|
1189
|
+
}
|
|
1190
|
+
export function getCjkTextUnits(text, profile, wordBreak) {
|
|
1191
|
+
const units = buildBaseCjkUnits(text, profile);
|
|
1192
|
+
return wordBreak === 'keep-all' ? mergeKeepAllTextUnits(text, units, profile) : units;
|
|
1193
|
+
}
|
|
1194
|
+
function isPreferredBreakGrapheme(grapheme) {
|
|
1195
|
+
return grapheme === '-' || grapheme === '\u058A' || grapheme === '\u2010' || grapheme === '\u2012' || grapheme === '\u2013' || grapheme === '\u2014';
|
|
1196
|
+
}
|
|
1197
|
+
export function isIndependentSymbolRun(text) {
|
|
1198
|
+
if (text.length === 0 || /\p{Cf}/u.test(text)) return false;
|
|
1199
|
+
for (const { segment } of getSharedGraphemeSegmenter().segment(text)){
|
|
1200
|
+
const base = String.fromCodePoint(segment.codePointAt(0));
|
|
1201
|
+
if (!/[\p{P}\p{S}]/u.test(base) || emojiPresentationRe.test(base) || segment.includes('\uFE0F') || /\p{Emoji_Modifier}/u.test(base)) return false;
|
|
1202
|
+
}
|
|
1203
|
+
return true;
|
|
1204
|
+
}
|
|
1205
|
+
export function getBreakablePreferredBreaks(text) {
|
|
1206
|
+
if (!/[-\u058A\u2010\u2012\u2013\u2014]/u.test(text)) return null;
|
|
1207
|
+
const breaks = [];
|
|
1208
|
+
let graphemeIndex = 0;
|
|
1209
|
+
for (const gs of getSharedGraphemeSegmenter().segment(text)){
|
|
1210
|
+
graphemeIndex++;
|
|
1211
|
+
const numericSign = gs.segment === '-' && isNumericHyphen(text, gs.index);
|
|
1212
|
+
if (isPreferredBreakGrapheme(gs.segment) && !numericSign) breaks.push(graphemeIndex);
|
|
1213
|
+
}
|
|
1214
|
+
return breaks.length === 0 ? null : breaks;
|
|
1215
|
+
}
|
|
1216
|
+
export function analyzeText(text, profile, whiteSpace = 'normal', wordBreak = 'normal') {
|
|
1217
|
+
const normalized = whiteSpace === 'pre-wrap' ? normalizeWhitespacePreWrap(text) : normalizeWhitespaceNormal(text);
|
|
1218
|
+
if (normalized.length === 0) {
|
|
1219
|
+
return {
|
|
1220
|
+
normalized,
|
|
1221
|
+
len: 0,
|
|
1222
|
+
texts: [],
|
|
1223
|
+
isWordLike: [],
|
|
1224
|
+
kinds: [],
|
|
1225
|
+
starts: []
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
const mergedSegmentation = buildMergedSegmentation(normalized, profile, whiteSpace);
|
|
1229
|
+
const segmentation = wordBreak === 'keep-all' ? mergeKeepAllTextSegments(normalized, mergedSegmentation, profile) : mergedSegmentation;
|
|
1230
|
+
return {
|
|
1231
|
+
normalized,
|
|
1232
|
+
...segmentation
|
|
1233
|
+
};
|
|
1234
|
+
}
|