tgmark 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tgmark/__init__.py +25 -0
- tgmark/conversion.py +196 -0
- tgmark/exceptions.py +42 -0
- tgmark/fit/__init__.py +1 -0
- tgmark/fit/graphemes.py +222 -0
- tgmark/fit/parts.py +270 -0
- tgmark/lower/__init__.py +1 -0
- tgmark/lower/characters.py +87 -0
- tgmark/lower/table.py +26 -0
- tgmark/model/__init__.py +1 -0
- tgmark/model/diagnostics.py +76 -0
- tgmark/model/document.py +84 -0
- tgmark/model/formatted.py +84 -0
- tgmark/model/server.py +41 -0
- tgmark/parse/__init__.py +1 -0
- tgmark/parse/decoding.py +24 -0
- tgmark/parse/plain.py +26 -0
- tgmark/py.typed +0 -0
- tgmark-0.1.0.dist-info/METADATA +111 -0
- tgmark-0.1.0.dist-info/RECORD +22 -0
- tgmark-0.1.0.dist-info/WHEEL +4 -0
- tgmark-0.1.0.dist-info/licenses/LICENSE +21 -0
tgmark/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Telegram Bot API markup converter for HTML, Markdown, and plain text.
|
|
2
|
+
|
|
3
|
+
Public surface
|
|
4
|
+
--------------
|
|
5
|
+
- `convert`: converts text into messages that the Bot API server keeps as sent.
|
|
6
|
+
- `Conversion`: the messages and the diagnostics of one conversion.
|
|
7
|
+
- `Message`: one message's text and its Bot API parameters.
|
|
8
|
+
- `Diagnostic`: a condition lenient mode repaired or dropped, and where it starts.
|
|
9
|
+
- `TGMarkError`: root of the exception hierarchy.
|
|
10
|
+
- `ConversionError`: raised in strict mode for the earliest such condition.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from tgmark.conversion import Conversion, Diagnostic, Message, convert
|
|
14
|
+
from tgmark.exceptions import ConversionError, TGMarkError
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"Conversion",
|
|
20
|
+
"ConversionError",
|
|
21
|
+
"Diagnostic",
|
|
22
|
+
"Message",
|
|
23
|
+
"TGMarkError",
|
|
24
|
+
"convert",
|
|
25
|
+
]
|
tgmark/conversion.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Conversion of text into Telegram messages: `convert` and the types it returns.
|
|
2
|
+
|
|
3
|
+
This module alone wires the stages together, `parse`, then `lower`, then `fit`, so no
|
|
4
|
+
domain imports another. The domains report each condition they repair or drop as a
|
|
5
|
+
foundation record, which this module maps onto `Diagnostic`; in strict mode it raises
|
|
6
|
+
for the record at the earliest position instead.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any, Literal
|
|
11
|
+
|
|
12
|
+
from tgmark.exceptions import ConversionError
|
|
13
|
+
from tgmark.fit.parts import fit_text
|
|
14
|
+
from tgmark.lower.table import lower_document
|
|
15
|
+
from tgmark.model.diagnostics import Notice
|
|
16
|
+
from tgmark.model.server import MESSAGE_LENGTH
|
|
17
|
+
from tgmark.parse.decoding import replace_surrogates
|
|
18
|
+
from tgmark.parse.plain import read_plain
|
|
19
|
+
|
|
20
|
+
# The values `source` and `target` take.
|
|
21
|
+
_SOURCES = ("plain",)
|
|
22
|
+
_TARGETS = ("entities",)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Diagnostic:
|
|
27
|
+
"""A condition in the input that lenient mode repaired or dropped.
|
|
28
|
+
|
|
29
|
+
Attributes:
|
|
30
|
+
code: The condition's code, such as `control-character`, which never changes
|
|
31
|
+
once released.
|
|
32
|
+
message: The condition and what lenient mode did instead, in one clause that
|
|
33
|
+
never quotes the input.
|
|
34
|
+
line: The line where the condition starts in the input, counted from 1.
|
|
35
|
+
column: The column in that line, counted from 1 in code points.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
code: str
|
|
39
|
+
message: str
|
|
40
|
+
line: int
|
|
41
|
+
column: int
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Message:
|
|
46
|
+
"""One Telegram message, as the Bot API server keeps it.
|
|
47
|
+
|
|
48
|
+
Attributes:
|
|
49
|
+
text: The text of the message.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
text: str
|
|
53
|
+
|
|
54
|
+
# The parameters' values differ in type from one name to the next, so `Any` lets
|
|
55
|
+
# them unpack into the keyword arguments of any Bot API client.
|
|
56
|
+
def params(self) -> dict[str, Any]:
|
|
57
|
+
"""Return the message's parameters for the Bot API's `sendMessage` method.
|
|
58
|
+
|
|
59
|
+
Each call returns a new `dict`, so a caller can add `chat_id` and the other
|
|
60
|
+
parameters of the call to it.
|
|
61
|
+
"""
|
|
62
|
+
return {"text": self.text}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Conversion:
|
|
67
|
+
"""What converting a text produced: its messages and its diagnostics.
|
|
68
|
+
|
|
69
|
+
Attributes:
|
|
70
|
+
messages: The messages, in input order; empty when nothing remains to send.
|
|
71
|
+
diagnostics: Each condition lenient mode repaired or dropped, in input order.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
messages: tuple[Message, ...]
|
|
75
|
+
diagnostics: tuple[Diagnostic, ...]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def convert(
|
|
79
|
+
text: str,
|
|
80
|
+
*,
|
|
81
|
+
source: Literal["plain"],
|
|
82
|
+
target: Literal["entities"],
|
|
83
|
+
max_length: int | None = None,
|
|
84
|
+
max_parts: int | None = None,
|
|
85
|
+
strict: bool = False,
|
|
86
|
+
) -> Conversion:
|
|
87
|
+
"""Convert `text` into Telegram messages that the Bot API server keeps as sent.
|
|
88
|
+
|
|
89
|
+
Lenient mode repairs or drops what the server would refuse or change and reports
|
|
90
|
+
each repair or drop as a diagnostic; strict mode raises at the earliest instead, for
|
|
91
|
+
a caller that refuses input that would not arrive unchanged. Text longer than one
|
|
92
|
+
message splits into several, each ending at the best break that fits. The options
|
|
93
|
+
are keyword-only so that a new option never changes the meaning of an existing call.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
text: The text to convert.
|
|
97
|
+
source: The format of `text`: `plain` reads every character as written.
|
|
98
|
+
target: The form of each message: `entities` sends its text with no parse mode.
|
|
99
|
+
max_length: The most code points the text of each message keeps, from 1 to
|
|
100
|
+
4,096; `None` for 4,096, the server's limit.
|
|
101
|
+
max_parts: The most messages to return, 1 or more; `None` for no cap. The text
|
|
102
|
+
past the last of them is dropped.
|
|
103
|
+
strict: Whether to raise `ConversionError` instead of repairing or dropping.
|
|
104
|
+
|
|
105
|
+
Raises:
|
|
106
|
+
TypeError: If `text`, `source`, or `target` is not a `str`, `max_length` or
|
|
107
|
+
`max_parts` is neither an `int` nor `None`, or `strict` is not a `bool`.
|
|
108
|
+
ValueError: If `source` or `target` names a format other than those above, or
|
|
109
|
+
`max_length` or `max_parts` is out of its range.
|
|
110
|
+
ConversionError: If `strict` is set and lenient mode would repair or
|
|
111
|
+
drop anything.
|
|
112
|
+
"""
|
|
113
|
+
_check_arguments(text, source, target, max_length, max_parts, strict)
|
|
114
|
+
|
|
115
|
+
valid, decoding = replace_surrogates(text)
|
|
116
|
+
formatted, lowering = lower_document(read_plain(valid))
|
|
117
|
+
parts, fitting = fit_text(
|
|
118
|
+
formatted,
|
|
119
|
+
max_length=MESSAGE_LENGTH if max_length is None else max_length,
|
|
120
|
+
max_parts=max_parts,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
notices = sorted((*decoding, *lowering, *fitting), key=lambda n: n.position)
|
|
124
|
+
if strict and notices:
|
|
125
|
+
raise _error(notices[0])
|
|
126
|
+
|
|
127
|
+
return Conversion(
|
|
128
|
+
messages=tuple(Message(part.text) for part in parts),
|
|
129
|
+
diagnostics=tuple(_diagnostic(notice) for notice in notices),
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _check_arguments(
|
|
134
|
+
text: object,
|
|
135
|
+
source: object,
|
|
136
|
+
target: object,
|
|
137
|
+
max_length: object,
|
|
138
|
+
max_parts: object,
|
|
139
|
+
strict: object,
|
|
140
|
+
) -> None:
|
|
141
|
+
"""Raise for an argument of `convert` of the wrong type or value."""
|
|
142
|
+
if not isinstance(text, str):
|
|
143
|
+
raise TypeError(f"text must be str, not {type(text).__name__}")
|
|
144
|
+
_check_choice("source", source, _SOURCES)
|
|
145
|
+
_check_choice("target", target, _TARGETS)
|
|
146
|
+
_check_count("max_length", max_length, MESSAGE_LENGTH)
|
|
147
|
+
_check_count("max_parts", max_parts, None)
|
|
148
|
+
if not isinstance(strict, bool):
|
|
149
|
+
raise TypeError(f"strict must be bool, not {type(strict).__name__}")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _check_choice(name: str, value: object, choices: tuple[str, ...]) -> None:
|
|
153
|
+
"""Raise unless `value`, the argument called `name`, is one of `choices`."""
|
|
154
|
+
if not isinstance(value, str):
|
|
155
|
+
raise TypeError(f"{name} must be str, not {type(value).__name__}")
|
|
156
|
+
if value not in choices:
|
|
157
|
+
alternatives = " or ".join(f"'{choice}'" for choice in choices)
|
|
158
|
+
raise ValueError(f"{name} must be {alternatives}")
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _check_count(name: str, value: object, highest: int | None) -> None:
|
|
162
|
+
"""Raise unless `value`, the argument called `name`, is `None` or a count.
|
|
163
|
+
|
|
164
|
+
A count is an `int` from 1 up to `highest`, or with no upper bound when `highest` is
|
|
165
|
+
`None`. A `bool` is refused although it is an `int`, since `True` would read as 1.
|
|
166
|
+
"""
|
|
167
|
+
if value is None:
|
|
168
|
+
return
|
|
169
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
170
|
+
raise TypeError(f"{name} must be int or None, not {type(value).__name__}")
|
|
171
|
+
if highest is None and value < 1:
|
|
172
|
+
raise ValueError(f"{name} must be at least 1")
|
|
173
|
+
if highest is not None and not 1 <= value <= highest:
|
|
174
|
+
raise ValueError(f"{name} must be from 1 to {highest}")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _diagnostic(notice: Notice) -> Diagnostic:
|
|
178
|
+
"""Return the diagnostic that reports `notice`."""
|
|
179
|
+
condition = notice.condition
|
|
180
|
+
return Diagnostic(
|
|
181
|
+
code=condition.code,
|
|
182
|
+
message=f"{condition.name} {condition.outcome}",
|
|
183
|
+
line=notice.position.line,
|
|
184
|
+
column=notice.position.column,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _error(notice: Notice) -> ConversionError:
|
|
189
|
+
"""Return the error strict mode raises for `notice`."""
|
|
190
|
+
line, column = notice.position.line, notice.position.column
|
|
191
|
+
return ConversionError(
|
|
192
|
+
f"{notice.condition.name} at line {line}, column {column}",
|
|
193
|
+
code=notice.condition.code,
|
|
194
|
+
line=line,
|
|
195
|
+
column=column,
|
|
196
|
+
)
|
tgmark/exceptions.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Exception hierarchy rooted at `TGMarkError`."""
|
|
2
|
+
|
|
3
|
+
from functools import partial
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TGMarkError(Exception):
|
|
7
|
+
"""Root of the exceptions the package defines.
|
|
8
|
+
|
|
9
|
+
Each exception class the package defines derives from it, so one `except` clause
|
|
10
|
+
catches them all.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ConversionError(TGMarkError):
|
|
15
|
+
"""Raised in strict mode where lenient mode would repair or drop the input.
|
|
16
|
+
|
|
17
|
+
Strict mode converts as lenient mode does, then raises for the condition at the
|
|
18
|
+
earliest input position, the one the input's author fixes first, instead of
|
|
19
|
+
returning a conversion that changes the input. The message names the condition and
|
|
20
|
+
its position and never quotes the input.
|
|
21
|
+
|
|
22
|
+
Attributes:
|
|
23
|
+
code: The condition's code, as a `Diagnostic` carries it.
|
|
24
|
+
line: The line where the condition starts in the input, counted from 1.
|
|
25
|
+
column: The column in that line, counted from 1 in code points.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, message: str, *, code: str, line: int, column: int) -> None:
|
|
29
|
+
super().__init__(message)
|
|
30
|
+
self.code = code
|
|
31
|
+
self.line = line
|
|
32
|
+
self.column = column
|
|
33
|
+
|
|
34
|
+
def __reduce__(
|
|
35
|
+
self,
|
|
36
|
+
) -> tuple[partial["ConversionError"], tuple[object, ...], dict[str, object]]:
|
|
37
|
+
# Unpickling calls the class with `args` alone, which lack the keyword-only
|
|
38
|
+
# attributes, so they travel in a partial.
|
|
39
|
+
rebuild = partial(
|
|
40
|
+
type(self), code=self.code, line=self.line, column=self.column
|
|
41
|
+
)
|
|
42
|
+
return rebuild, self.args, self.__dict__
|
tgmark/fit/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Fitting of formatted text into messages."""
|
tgmark/fit/graphemes.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Grapheme cluster boundaries a cut may use, so that no cut splits a character.
|
|
2
|
+
|
|
3
|
+
The rule follows UAX #29's extended grapheme clusters from `unicodedata` and small fixed
|
|
4
|
+
sets of code points, since the package ships none of Unicode's property files. Where
|
|
5
|
+
`unicodedata` cannot settle a boundary, the rule refuses it: a refused boundary costs at
|
|
6
|
+
most a shorter message, while a cut inside a cluster splits what a reader sees as one
|
|
7
|
+
character. The fixed sets hold what the oldest supported Python's Unicode version,
|
|
8
|
+
14.0.0, lacks up to 18.0.0, so every supported Python refuses each cut that its own
|
|
9
|
+
version or 18.0.0 forbids.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import unicodedata
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _chars(*spans: int | tuple[int, int]) -> frozenset[str]:
|
|
17
|
+
"""Return the characters of `spans`, each a code point or an inclusive range."""
|
|
18
|
+
codes: list[Iterable[int]] = [
|
|
19
|
+
range(span[0], span[1] + 1) if isinstance(span, tuple) else (span,)
|
|
20
|
+
for span in spans
|
|
21
|
+
]
|
|
22
|
+
return frozenset(chr(code) for group in codes for code in group)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# The general categories of marks, which attach to the character before them
|
|
26
|
+
# (GB9, GB9a).
|
|
27
|
+
_MARKS = frozenset({"Mn", "Mc", "Me"})
|
|
28
|
+
|
|
29
|
+
# The characters that attach to the character before them without being marks: the
|
|
30
|
+
# joiner U+200D, and the characters of GraphemeBreakProperty.txt, Extend and
|
|
31
|
+
# SpacingMark, whose category is not a mark's, the emoji modifiers among them
|
|
32
|
+
# (GB9, GB9a).
|
|
33
|
+
_ATTACHING = _chars(
|
|
34
|
+
0x0E33,
|
|
35
|
+
0x0EB3,
|
|
36
|
+
0x200C,
|
|
37
|
+
0x200D,
|
|
38
|
+
(0xFF9E, 0xFF9F),
|
|
39
|
+
(0x1F3FB, 0x1F3FF),
|
|
40
|
+
(0xE0020, 0xE007F),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# The characters assigned after Unicode 14.0.0, up to 18.0.0, that attach to the
|
|
44
|
+
# character before them (GraphemeBreakProperty.txt, Extend and SpacingMark), for a
|
|
45
|
+
# Python whose `unicodedata` predates them.
|
|
46
|
+
_ADDED_ATTACHING = _chars(
|
|
47
|
+
(0x05C8, 0x05C9),
|
|
48
|
+
0x0897,
|
|
49
|
+
(0x0B53, 0x0B54),
|
|
50
|
+
0x0CF3,
|
|
51
|
+
0x0ECE,
|
|
52
|
+
(0x1ACF, 0x1AF0),
|
|
53
|
+
(0x10D69, 0x10D6D),
|
|
54
|
+
(0x10ECB, 0x10ECF),
|
|
55
|
+
(0x10EF0, 0x10EFF),
|
|
56
|
+
0x11241,
|
|
57
|
+
(0x113B8, 0x113C0),
|
|
58
|
+
0x113C2,
|
|
59
|
+
0x113C5,
|
|
60
|
+
(0x113C7, 0x113CA),
|
|
61
|
+
(0x113CC, 0x113D0),
|
|
62
|
+
0x113D2,
|
|
63
|
+
(0x113E1, 0x113E2),
|
|
64
|
+
(0x11B60, 0x11B67),
|
|
65
|
+
0x11DF0,
|
|
66
|
+
(0x11F00, 0x11F01),
|
|
67
|
+
0x11F03,
|
|
68
|
+
(0x11F34, 0x11F3A),
|
|
69
|
+
(0x11F3E, 0x11F42),
|
|
70
|
+
0x11F5A,
|
|
71
|
+
0x13440,
|
|
72
|
+
(0x13447, 0x13455),
|
|
73
|
+
(0x1611E, 0x1612F),
|
|
74
|
+
(0x1D127, 0x1D128),
|
|
75
|
+
(0x1D250, 0x1D252),
|
|
76
|
+
(0x1D25B, 0x1D25C),
|
|
77
|
+
0x1D25F,
|
|
78
|
+
(0x1D280, 0x1D281),
|
|
79
|
+
0x1E08F,
|
|
80
|
+
(0x1E4EC, 0x1E4EF),
|
|
81
|
+
(0x1E5EE, 0x1E5EF),
|
|
82
|
+
0x1E6E3,
|
|
83
|
+
0x1E6E6,
|
|
84
|
+
(0x1E6EE, 0x1E6EF),
|
|
85
|
+
0x1E6F5,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# The characters that attach to the character after them: the joiner U+200D (GB11), and
|
|
89
|
+
# the prepend characters of every Unicode version from 14.0.0 to 18.0.0
|
|
90
|
+
# (GraphemeBreakProperty.txt, Prepend; GB9b).
|
|
91
|
+
_JOINING = _chars(
|
|
92
|
+
(0x0600, 0x0605),
|
|
93
|
+
0x06DD,
|
|
94
|
+
0x070F,
|
|
95
|
+
(0x0890, 0x0891),
|
|
96
|
+
0x08E2,
|
|
97
|
+
0x0D4E,
|
|
98
|
+
0x200D,
|
|
99
|
+
0x110BD,
|
|
100
|
+
0x110CD,
|
|
101
|
+
(0x111C2, 0x111C3),
|
|
102
|
+
0x113D1,
|
|
103
|
+
0x1193F,
|
|
104
|
+
0x11941,
|
|
105
|
+
0x11A3A,
|
|
106
|
+
(0x11A84, 0x11A89),
|
|
107
|
+
0x11D46,
|
|
108
|
+
0x11F02,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
# The linkers that a combining class of 9 does not mark: the letters 18.0.0 links to the
|
|
112
|
+
# consonant after them, and the viramas assigned after 14.0.0
|
|
113
|
+
# (DerivedCoreProperties.txt, InCB=Linker; GB9c).
|
|
114
|
+
_LINKERS = _chars(0x1CF5, 0x1CF6, 0x11A3A, 0x113D0, 0x11F42)
|
|
115
|
+
|
|
116
|
+
# The parts of Hangul syllables, the Kirat Rai vowel signs joining as Hangul vowels do
|
|
117
|
+
# (GraphemeBreakProperty.txt, L, V, and T; GB6, GB7, GB8).
|
|
118
|
+
_LEADING = _chars((0x1100, 0x115F), (0xA960, 0xA97C))
|
|
119
|
+
_VOWELS = _chars((0x1160, 0x11A7), (0xD7B0, 0xD7C6), 0x16D63, (0x16D67, 0x16D6A))
|
|
120
|
+
_TRAILING = _chars((0x11A8, 0x11FF), (0xD7CB, 0xD7FB))
|
|
121
|
+
_SYLLABLES = range(0xAC00, 0xD7A4)
|
|
122
|
+
_TRAILS = 28
|
|
123
|
+
|
|
124
|
+
# The regional indicators, which pair into flags (GB12, GB13).
|
|
125
|
+
_REGIONAL = _chars((0x1F1E6, 0x1F1FF))
|
|
126
|
+
|
|
127
|
+
# The bits of a class, each a property the rule reads from a character, and the Hangul
|
|
128
|
+
# part above them.
|
|
129
|
+
_ATTACHES = 1 << 0
|
|
130
|
+
_JOINS = 1 << 1
|
|
131
|
+
_LINKS = 1 << 2
|
|
132
|
+
_LETTER = 1 << 3
|
|
133
|
+
_PAIRS = 1 << 4
|
|
134
|
+
_CR = 1 << 5
|
|
135
|
+
_LF = 1 << 6
|
|
136
|
+
_PART = 7
|
|
137
|
+
_L, _V, _T, _LV, _LVT = range(1, 6)
|
|
138
|
+
|
|
139
|
+
# The pairs of Hangul parts that stay together.
|
|
140
|
+
_HANGUL_PAIRS = frozenset(
|
|
141
|
+
{
|
|
142
|
+
*((_L, after) for after in (_L, _V, _LV, _LVT)),
|
|
143
|
+
*((before, after) for before in (_LV, _V) for after in (_V, _T)),
|
|
144
|
+
*((before, _T) for before in (_LVT, _T)),
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def classify(char: str) -> int:
|
|
150
|
+
"""Return the class of `char`: a bit for each property the rule reads from it.
|
|
151
|
+
|
|
152
|
+
Two characters of one class get the same verdict at every offset, which lets a test
|
|
153
|
+
check one character of each class in place of every code point.
|
|
154
|
+
"""
|
|
155
|
+
category = unicodedata.category(char)
|
|
156
|
+
kind = 0
|
|
157
|
+
if category in _MARKS or char in _ATTACHING or char in _ADDED_ATTACHING:
|
|
158
|
+
kind |= _ATTACHES
|
|
159
|
+
# Only marks carry a combining class of 9, which viramas have.
|
|
160
|
+
if unicodedata.combining(char) == 9:
|
|
161
|
+
kind |= _LINKS
|
|
162
|
+
if char in _JOINING:
|
|
163
|
+
kind |= _JOINS
|
|
164
|
+
if char in _LINKERS:
|
|
165
|
+
kind |= _LINKS
|
|
166
|
+
# A code point unassigned in the running Python may be a consonant in a later
|
|
167
|
+
# version, so it counts as a letter.
|
|
168
|
+
if category[0] == "L" or category == "Cn":
|
|
169
|
+
kind |= _LETTER
|
|
170
|
+
if char in _REGIONAL:
|
|
171
|
+
kind |= _PAIRS
|
|
172
|
+
if char == "\r":
|
|
173
|
+
kind |= _CR
|
|
174
|
+
if char == "\n":
|
|
175
|
+
kind |= _LF
|
|
176
|
+
return kind | _hangul(char) << _PART
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def boundaries(text: str) -> bytes:
|
|
180
|
+
"""Return a flag per offset of `text`, 1 where a cut keeps every cluster whole.
|
|
181
|
+
|
|
182
|
+
The flags run from offset 0 to `len(text)`, both ends included and set, so a cut at
|
|
183
|
+
an offset is allowed exactly when its flag is 1. A cut is refused inside CR LF;
|
|
184
|
+
before a character that attaches to the one before it, such as a mark; after the
|
|
185
|
+
joiner U+200D or a prepend character; after a linker, and any attaching characters
|
|
186
|
+
after it, before a letter or an unassigned code point; inside a Hangul syllable
|
|
187
|
+
sequence; and inside a pair of regional indicators.
|
|
188
|
+
"""
|
|
189
|
+
flags = bytearray(b"\x01" * (len(text) + 1))
|
|
190
|
+
before = 0
|
|
191
|
+
linked = False
|
|
192
|
+
regional = 0
|
|
193
|
+
for offset, char in enumerate(text):
|
|
194
|
+
kind = classify(char)
|
|
195
|
+
if offset and (
|
|
196
|
+
kind & _ATTACHES
|
|
197
|
+
or before & _JOINS
|
|
198
|
+
or (before & _CR and kind & _LF)
|
|
199
|
+
or (linked and kind & _LETTER)
|
|
200
|
+
or (before >> _PART, kind >> _PART) in _HANGUL_PAIRS
|
|
201
|
+
or (kind & _PAIRS and regional % 2 == 1)
|
|
202
|
+
):
|
|
203
|
+
flags[offset] = 0
|
|
204
|
+
|
|
205
|
+
linked = bool(kind & _LINKS) or (linked and bool(kind & _ATTACHES))
|
|
206
|
+
regional = regional + 1 if kind & _PAIRS else 0
|
|
207
|
+
before = kind
|
|
208
|
+
|
|
209
|
+
return bytes(flags)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _hangul(char: str) -> int:
|
|
213
|
+
"""Return the part of a Hangul syllable sequence that `char` is, or 0 for none."""
|
|
214
|
+
if char in _LEADING:
|
|
215
|
+
return _L
|
|
216
|
+
if char in _VOWELS:
|
|
217
|
+
return _V
|
|
218
|
+
if char in _TRAILING:
|
|
219
|
+
return _T
|
|
220
|
+
if ord(char) in _SYLLABLES:
|
|
221
|
+
return _LV if (ord(char) - _SYLLABLES.start) % _TRAILS == 0 else _LVT
|
|
222
|
+
return 0
|