cellstr 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cellstr/__init__.py +41 -0
- cellstr/__main__.py +8 -0
- cellstr/_ansi.py +62 -0
- cellstr/_cells.py +113 -0
- cellstr/_cli.py +184 -0
- cellstr/_graphemes.py +101 -0
- cellstr/_ops.py +147 -0
- cellstr/_props.py +61 -0
- cellstr/_truncate.py +130 -0
- cellstr/_ucd_data.py +31 -0
- cellstr/_ucd_part1.py +166 -0
- cellstr/_width.py +111 -0
- cellstr/py.typed +0 -0
- cellstr-0.2.1.dist-info/METADATA +144 -0
- cellstr-0.2.1.dist-info/RECORD +19 -0
- cellstr-0.2.1.dist-info/WHEEL +4 -0
- cellstr-0.2.1.dist-info/entry_points.txt +2 -0
- cellstr-0.2.1.dist-info/licenses/LICENSE +21 -0
- cellstr-0.2.1.dist-info/licenses/LICENSE-UNICODE +39 -0
cellstr/__init__.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Display-cell-accurate length, slicing, padding and truncation for terminal text.
|
|
2
|
+
|
|
3
|
+
Every function measures what a terminal draws: ANSI escape sequences count as nothing,
|
|
4
|
+
East Asian wide characters as two cells, combining marks and other zero-width code
|
|
5
|
+
points as nothing, and an emoji sequence or a flag as one two-cell glyph. Width data
|
|
6
|
+
comes from the Unicode Character Database, version ``UNICODE_VERSION``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from ._ansi import strip_ansi
|
|
12
|
+
from ._cells import Cell, cell_len, expand_tabs, iter_cells
|
|
13
|
+
from ._graphemes import iter_graphemes
|
|
14
|
+
from ._ops import cell_slice, center, ljust, rjust
|
|
15
|
+
from ._props import UNICODE_VERSION
|
|
16
|
+
from ._truncate import Align, Position, Truncation, fit, truncate, truncate_detailed
|
|
17
|
+
from ._width import char_width, grapheme_width
|
|
18
|
+
|
|
19
|
+
__version__ = "0.2.1"
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"UNICODE_VERSION",
|
|
23
|
+
"Align",
|
|
24
|
+
"Cell",
|
|
25
|
+
"Position",
|
|
26
|
+
"Truncation",
|
|
27
|
+
"cell_len",
|
|
28
|
+
"cell_slice",
|
|
29
|
+
"center",
|
|
30
|
+
"char_width",
|
|
31
|
+
"expand_tabs",
|
|
32
|
+
"fit",
|
|
33
|
+
"grapheme_width",
|
|
34
|
+
"iter_cells",
|
|
35
|
+
"iter_graphemes",
|
|
36
|
+
"ljust",
|
|
37
|
+
"rjust",
|
|
38
|
+
"strip_ansi",
|
|
39
|
+
"truncate",
|
|
40
|
+
"truncate_detailed",
|
|
41
|
+
]
|
cellstr/__main__.py
ADDED
cellstr/_ansi.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""ANSI escape sequence detection and removal."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import NamedTuple
|
|
7
|
+
|
|
8
|
+
# One alternative per ECMA-48 escape family, longest first so a string introducer is never
|
|
9
|
+
# mistaken for a two-character escape:
|
|
10
|
+
# CSI ESC [ params intermediates final colours (SGR), cursor moves
|
|
11
|
+
# OSC ESC ] ... BEL or ST titles, OSC 8 hyperlinks
|
|
12
|
+
# strings ESC P / X / ^ / _ ... ST DCS, SOS, PM, APC
|
|
13
|
+
# nF ESC intermediates final charset designation such as ESC ( B
|
|
14
|
+
# Fp/Fe/Fs ESC one final byte ESC 7, ESC 8, ESC =, ESC c, ESC M
|
|
15
|
+
# An unterminated OSC or string sequence is never swallowed to the end of the text: at most
|
|
16
|
+
# its introducer counts as an escape and the rest is measured as text, so nothing visible
|
|
17
|
+
# silently disappears.
|
|
18
|
+
_ANSI_RE = re.compile(
|
|
19
|
+
r"\x1b(?:"
|
|
20
|
+
r"\[[0-?]*[ -/]*[@-~]"
|
|
21
|
+
r"|\][^\x07\x1b]*(?:\x07|\x1b\\)"
|
|
22
|
+
r"|[PX^_][^\x1b]*\x1b\\"
|
|
23
|
+
r"|[ -/]+[0-~]"
|
|
24
|
+
r"|[0-Z\\`-~]"
|
|
25
|
+
r")"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def strip_ansi(text: str) -> str:
|
|
30
|
+
"""Return ``text`` with every ANSI escape sequence removed.
|
|
31
|
+
|
|
32
|
+
Recognised: CSI sequences (SGR colours and styles, cursor movement), OSC sequences
|
|
33
|
+
ended by BEL or ST (window titles, OSC 8 hyperlinks), DCS, SOS, PM and APC strings,
|
|
34
|
+
charset designations and the single-byte escapes.
|
|
35
|
+
|
|
36
|
+
>>> strip_ansi("\\x1b[1;31merror\\x1b[0m: \\x1b]8;;https://example.com\\x07docs\\x1b]8;;\\x07")
|
|
37
|
+
'error: docs'
|
|
38
|
+
"""
|
|
39
|
+
return _ANSI_RE.sub("", text)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Segment(NamedTuple):
|
|
43
|
+
"""A run of ``text`` tagged as either an escape sequence or printable text."""
|
|
44
|
+
|
|
45
|
+
text: str
|
|
46
|
+
is_escape: bool
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def segments(text: str) -> list[Segment]:
|
|
50
|
+
"""Split ``text`` into ordered printable and escape segments."""
|
|
51
|
+
if "\x1b" not in text:
|
|
52
|
+
return [Segment(text, False)] if text else []
|
|
53
|
+
out: list[Segment] = []
|
|
54
|
+
last = 0
|
|
55
|
+
for match in _ANSI_RE.finditer(text):
|
|
56
|
+
if match.start() > last:
|
|
57
|
+
out.append(Segment(text[last : match.start()], False))
|
|
58
|
+
out.append(Segment(match.group(), True))
|
|
59
|
+
last = match.end()
|
|
60
|
+
if last < len(text):
|
|
61
|
+
out.append(Segment(text[last:], False))
|
|
62
|
+
return out
|
cellstr/_cells.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Typed iteration over the cells of terminal text, measuring and tab expansion."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
from typing import NamedTuple
|
|
7
|
+
|
|
8
|
+
from ._ansi import segments
|
|
9
|
+
from ._graphemes import grapheme_ends
|
|
10
|
+
from ._width import check_ambiguous, cluster_width, text_width
|
|
11
|
+
|
|
12
|
+
_LINE_BREAKS = frozenset({"\n", "\r", "\r\n"})
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Cell(NamedTuple):
|
|
16
|
+
"""One drawable unit of terminal text, or one escape sequence.
|
|
17
|
+
|
|
18
|
+
``text`` is a whole grapheme cluster or a whole escape sequence, ``width`` the cells
|
|
19
|
+
it occupies (always 0 for an escape), ``column`` the cell offset where it starts,
|
|
20
|
+
counted from the start of the text, and ``escape`` tells the two kinds apart.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
text: str
|
|
24
|
+
width: int
|
|
25
|
+
column: int
|
|
26
|
+
escape: bool
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def check_tabsize(tabsize: int | None) -> int | None:
|
|
30
|
+
"""Return ``tabsize`` after checking it is None or a positive integer."""
|
|
31
|
+
if tabsize is not None and (type(tabsize) is not int or tabsize < 1):
|
|
32
|
+
raise ValueError(f"tabsize must be a positive integer or None, not {tabsize!r}")
|
|
33
|
+
return tabsize
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def iter_cells(text: str, *, ambiguous: int = 1, tabsize: int | None = None) -> Iterator[Cell]:
|
|
37
|
+
"""Yield the escape sequences and grapheme clusters of ``text`` with their widths.
|
|
38
|
+
|
|
39
|
+
With ``tabsize`` set, a tab advances to the next multiple of ``tabsize`` counted from
|
|
40
|
+
the start of the current line (a CR or LF starts a new line); otherwise a tab is a
|
|
41
|
+
zero-width control like every other one. The texts of the cells concatenate back to
|
|
42
|
+
``text`` exactly.
|
|
43
|
+
|
|
44
|
+
>>> [(len(c.text), c.width, c.column) for c in iter_cells("a\\u4f60\\U0001f1ea\\U0001f1f8")]
|
|
45
|
+
[(1, 1, 0), (1, 2, 1), (2, 2, 3)]
|
|
46
|
+
"""
|
|
47
|
+
check_ambiguous(ambiguous)
|
|
48
|
+
check_tabsize(tabsize)
|
|
49
|
+
column = 0
|
|
50
|
+
line_start = 0
|
|
51
|
+
for segment in segments(text):
|
|
52
|
+
if segment.is_escape:
|
|
53
|
+
yield Cell(segment.text, 0, column, True)
|
|
54
|
+
continue
|
|
55
|
+
part = segment.text
|
|
56
|
+
start = 0
|
|
57
|
+
for end in grapheme_ends(part):
|
|
58
|
+
cluster = part[start:end]
|
|
59
|
+
start = end
|
|
60
|
+
if tabsize is not None and cluster == "\t":
|
|
61
|
+
width = tabsize - (column - line_start) % tabsize
|
|
62
|
+
else:
|
|
63
|
+
width = cluster_width(cluster, ambiguous)
|
|
64
|
+
yield Cell(cluster, width, column, False)
|
|
65
|
+
column += width
|
|
66
|
+
if cluster in _LINE_BREAKS:
|
|
67
|
+
line_start = column
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def cell_len(text: str, *, ambiguous: int = 1, tabsize: int | None = None) -> int:
|
|
71
|
+
"""Number of display cells ``text`` occupies, ignoring ANSI escape sequences.
|
|
72
|
+
|
|
73
|
+
Clusters are measured whole (see ``grapheme_width``), so an emoji ZWJ sequence or a
|
|
74
|
+
flag counts once; an escape sequence ends the cluster before it. Controls, including
|
|
75
|
+
CR and LF, add nothing, so this is the width of a single line. ``ambiguous`` and
|
|
76
|
+
``tabsize`` are explained in ``char_width`` and ``iter_cells``.
|
|
77
|
+
|
|
78
|
+
>>> cell_len("\\x1b[31mWARN\\x1b[0m"), cell_len("\\u4f60\\u597d"), cell_len("\\u00e9")
|
|
79
|
+
(4, 4, 1)
|
|
80
|
+
>>> cell_len("\\U0001f469\\u200d\\U0001f4bb"), cell_len("\\u03b1\\u03b2", ambiguous=2)
|
|
81
|
+
(2, 4)
|
|
82
|
+
>>> cell_len("a\\tb", tabsize=8)
|
|
83
|
+
9
|
|
84
|
+
"""
|
|
85
|
+
check_ambiguous(ambiguous)
|
|
86
|
+
if check_tabsize(tabsize) is None or "\t" not in text:
|
|
87
|
+
return text_width(text, ambiguous)
|
|
88
|
+
return sum(cell.width for cell in iter_cells(text, ambiguous=ambiguous, tabsize=tabsize))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def expand_tabs(text: str, tabsize: int = 8, *, ambiguous: int = 1) -> str:
|
|
92
|
+
"""Replace every tab with the spaces it covers on screen.
|
|
93
|
+
|
|
94
|
+
Unlike ``str.expandtabs`` the columns are display cells, so wide characters, emoji
|
|
95
|
+
and escape sequences before a tab do not throw it off. Escapes are kept in place.
|
|
96
|
+
|
|
97
|
+
>>> expand_tabs("\\u4f60\\tx", 4) == "\\u4f60 x"
|
|
98
|
+
True
|
|
99
|
+
"""
|
|
100
|
+
if check_tabsize(tabsize) is None:
|
|
101
|
+
raise ValueError("tabsize must be a positive integer")
|
|
102
|
+
if "\t" not in text:
|
|
103
|
+
check_ambiguous(ambiguous)
|
|
104
|
+
return text
|
|
105
|
+
return "".join(
|
|
106
|
+
" " * cell.width if cell.text == "\t" and not cell.escape else cell.text
|
|
107
|
+
for cell in iter_cells(text, ambiguous=ambiguous, tabsize=tabsize)
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def total_width(cells: list[Cell]) -> int:
|
|
112
|
+
"""Width of a list of cells produced by ``iter_cells``."""
|
|
113
|
+
return cells[-1].column + cells[-1].width if cells else 0
|
cellstr/_cli.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Command line interface: measure, truncate, pad, fit and slice stdin lines by cells."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
from typing import NoReturn
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from ._cells import cell_len
|
|
13
|
+
from ._ops import cell_slice, center, check_fill, ljust, rjust
|
|
14
|
+
from ._truncate import fit, truncate
|
|
15
|
+
|
|
16
|
+
EPILOG = """\
|
|
17
|
+
examples:
|
|
18
|
+
printf 'abc\\n\\344\\275\\240\\345\\245\\275\\n' | cellstr measure
|
|
19
|
+
printf 'abc\\n\\344\\275\\240\\345\\245\\275\\n' | cellstr measure --max
|
|
20
|
+
printf 'a long status message\\n' | cellstr truncate 10
|
|
21
|
+
printf '/usr/local/lib/python3/site-packages\\n' | cellstr truncate 16 --position middle
|
|
22
|
+
printf 'id\\n12345\\n' | cellstr pad 8 --align right
|
|
23
|
+
printf 'name\\na much longer name\\n' | cellstr fit 10
|
|
24
|
+
printf '\\x1b[31mred text\\x1b[0m\\n' | cellstr slice 4
|
|
25
|
+
|
|
26
|
+
Every command reads UTF-8 lines from stdin and writes one result line per input line,
|
|
27
|
+
keeping each line ending. ANSI escape sequences are ignored when measuring and kept in
|
|
28
|
+
the output. --ambiguous 2 counts East Asian Ambiguous characters (Greek, Cyrillic, box
|
|
29
|
+
drawing) as two cells, as CJK terminals draw them. Errors go to stderr with exit
|
|
30
|
+
status 1.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _Parser(argparse.ArgumentParser):
|
|
35
|
+
def error(self, message: str) -> NoReturn:
|
|
36
|
+
self.print_usage(sys.stderr)
|
|
37
|
+
self.exit(1, f"{self.prog}: error: {message}\n")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _cells(text: str) -> int:
|
|
41
|
+
try:
|
|
42
|
+
value = int(text)
|
|
43
|
+
except ValueError:
|
|
44
|
+
raise argparse.ArgumentTypeError(f"not an integer: {text!r}") from None
|
|
45
|
+
if value < 0:
|
|
46
|
+
raise argparse.ArgumentTypeError(f"must not be negative: {value}")
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _positive(text: str) -> int:
|
|
51
|
+
value = _cells(text)
|
|
52
|
+
if value == 0:
|
|
53
|
+
raise argparse.ArgumentTypeError("must be at least 1")
|
|
54
|
+
return value
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _add(sub: argparse._SubParsersAction[_Parser], name: str, text: str, example: str) -> _Parser:
|
|
58
|
+
common = _Parser(add_help=False)
|
|
59
|
+
common.add_argument(
|
|
60
|
+
"--ambiguous",
|
|
61
|
+
type=int,
|
|
62
|
+
choices=(1, 2),
|
|
63
|
+
default=1,
|
|
64
|
+
help="cells for East Asian Ambiguous characters (default 1)",
|
|
65
|
+
)
|
|
66
|
+
common.add_argument(
|
|
67
|
+
"--tabsize",
|
|
68
|
+
type=_positive,
|
|
69
|
+
default=None,
|
|
70
|
+
metavar="N",
|
|
71
|
+
help="expand tabs to stops every N cells (default: tabs count as zero)",
|
|
72
|
+
)
|
|
73
|
+
return sub.add_parser(
|
|
74
|
+
name,
|
|
75
|
+
parents=[common],
|
|
76
|
+
help=text,
|
|
77
|
+
description=text,
|
|
78
|
+
epilog=f"example:\n {example}\n",
|
|
79
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _cut_options(parser: _Parser) -> None:
|
|
84
|
+
parser.add_argument("-s", "--suffix", default="...", help="marks the cut (default ...)")
|
|
85
|
+
parser.add_argument(
|
|
86
|
+
"--position",
|
|
87
|
+
choices=("end", "start", "middle"),
|
|
88
|
+
default="end",
|
|
89
|
+
help="where to cut (default end)",
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _pad_options(parser: _Parser) -> None:
|
|
94
|
+
parser.add_argument(
|
|
95
|
+
"--align", choices=("left", "right", "center"), default="left", help="default left"
|
|
96
|
+
)
|
|
97
|
+
parser.add_argument("--fill", default=" ", help="one-cell padding character (default space)")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
101
|
+
"""Return the argument parser for the ``cellstr`` command."""
|
|
102
|
+
parser = _Parser(
|
|
103
|
+
prog="cellstr",
|
|
104
|
+
description="Measure, truncate, pad and slice text by terminal display cells.",
|
|
105
|
+
epilog=EPILOG,
|
|
106
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
107
|
+
)
|
|
108
|
+
parser.add_argument("--version", action="version", version=f"cellstr {__version__}")
|
|
109
|
+
sub = parser.add_subparsers(dest="command", required=True, parser_class=_Parser)
|
|
110
|
+
measure = _add(sub, "measure", "print the display width of each line", "cellstr measure")
|
|
111
|
+
measure.add_argument("--max", action="store_true", help="print only the widest width")
|
|
112
|
+
cut = _add(sub, "truncate", "cut each line to N cells", "cellstr truncate 20 --suffix '~'")
|
|
113
|
+
cut.add_argument("width", type=_cells, metavar="N", help="maximum width in cells")
|
|
114
|
+
_cut_options(cut)
|
|
115
|
+
pad = _add(sub, "pad", "pad each line to N cells", "cellstr pad 12 --align center")
|
|
116
|
+
pad.add_argument("width", type=_cells, metavar="N", help="target width in cells")
|
|
117
|
+
_pad_options(pad)
|
|
118
|
+
fitted = _add(sub, "fit", "truncate or pad each line to exactly N cells", "cellstr fit 12")
|
|
119
|
+
fitted.add_argument("width", type=_cells, metavar="N", help="exact width in cells")
|
|
120
|
+
_cut_options(fitted)
|
|
121
|
+
_pad_options(fitted)
|
|
122
|
+
piece = _add(sub, "slice", "keep the cells in columns [START, STOP)", "cellstr slice 2 10")
|
|
123
|
+
piece.add_argument("start", type=_cells, metavar="START", help="first column, from 0")
|
|
124
|
+
piece.add_argument("stop", type=_cells, nargs="?", metavar="STOP", help="end column")
|
|
125
|
+
return parser
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _lines(text: str) -> list[tuple[str, str]]:
|
|
129
|
+
"""Split ``text`` on LF into ``(body, line ending)`` pairs, keeping CR LF and LF."""
|
|
130
|
+
pairs: list[tuple[str, str]] = []
|
|
131
|
+
parts = text.split("\n")
|
|
132
|
+
for index, body in enumerate(parts):
|
|
133
|
+
last = index == len(parts) - 1
|
|
134
|
+
if last and not body:
|
|
135
|
+
break
|
|
136
|
+
end = "" if last else "\n"
|
|
137
|
+
if body.endswith("\r"):
|
|
138
|
+
body, end = body[:-1], "\r" + end
|
|
139
|
+
pairs.append((body, end))
|
|
140
|
+
return pairs
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _transform(args: argparse.Namespace) -> Callable[[str], str]:
|
|
144
|
+
policy = {"ambiguous": args.ambiguous, "tabsize": args.tabsize}
|
|
145
|
+
if args.command == "truncate":
|
|
146
|
+
return lambda s: truncate(s, args.width, args.suffix, position=args.position, **policy)
|
|
147
|
+
if args.command == "fit":
|
|
148
|
+
options = {"suffix": args.suffix, "position": args.position, "fillchar": args.fill}
|
|
149
|
+
return lambda s: fit(s, args.width, align=args.align, **options, **policy)
|
|
150
|
+
if args.command == "pad":
|
|
151
|
+
pad = {"left": ljust, "right": rjust, "center": center}[args.align]
|
|
152
|
+
return lambda s: pad(s, args.width, args.fill, **policy)
|
|
153
|
+
return lambda s: cell_slice(s, args.start, args.stop, **policy)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _run(args: argparse.Namespace, text: str) -> str:
|
|
157
|
+
lines = _lines(text)
|
|
158
|
+
if args.command == "measure":
|
|
159
|
+
widths = [cell_len(b, ambiguous=args.ambiguous, tabsize=args.tabsize) for b, _ in lines]
|
|
160
|
+
return f"{max(widths, default=0)}\n" if args.max else "".join(f"{w}\n" for w in widths)
|
|
161
|
+
if args.command in ("pad", "fit"):
|
|
162
|
+
check_fill(args.fill, args.ambiguous)
|
|
163
|
+
if args.command == "slice" and args.stop is not None and args.stop < args.start:
|
|
164
|
+
raise ValueError(f"STOP ({args.stop}) must not be less than START ({args.start})")
|
|
165
|
+
change = _transform(args)
|
|
166
|
+
return "".join(change(body) + end for body, end in lines)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
170
|
+
"""Run the command line tool and return its exit status."""
|
|
171
|
+
args = build_parser().parse_args(argv)
|
|
172
|
+
try:
|
|
173
|
+
result = _run(args, sys.stdin.buffer.read().decode("utf-8"))
|
|
174
|
+
except (ValueError, UnicodeDecodeError) as exc:
|
|
175
|
+
print(f"cellstr: error: {exc}", file=sys.stderr)
|
|
176
|
+
return 1
|
|
177
|
+
try:
|
|
178
|
+
sys.stdout.buffer.write(result.encode("utf-8", "surrogatepass"))
|
|
179
|
+
sys.stdout.buffer.flush()
|
|
180
|
+
except BrokenPipeError:
|
|
181
|
+
# The reader went away (for example `| head -1`); exit quietly like other filters.
|
|
182
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
183
|
+
os.dup2(devnull, sys.stdout.fileno())
|
|
184
|
+
return 0
|
cellstr/_graphemes.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Extended grapheme cluster segmentation per UAX #29, using embedded UCD tables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
|
|
7
|
+
from ._props import EXT_PICT_BIT, GCB, INCB_SHIFT, prop
|
|
8
|
+
|
|
9
|
+
_CR = GCB["CR"]
|
|
10
|
+
_LF = GCB["LF"]
|
|
11
|
+
_CONTROL = GCB["Control"]
|
|
12
|
+
_EXTEND = GCB["Extend"]
|
|
13
|
+
_ZWJ = GCB["ZWJ"]
|
|
14
|
+
_RI = GCB["Regional_Indicator"]
|
|
15
|
+
_PREPEND = GCB["Prepend"]
|
|
16
|
+
_SPACING = GCB["SpacingMark"]
|
|
17
|
+
_L = GCB["L"]
|
|
18
|
+
_V = GCB["V"]
|
|
19
|
+
_T = GCB["T"]
|
|
20
|
+
_LV = GCB["LV"]
|
|
21
|
+
_LVT = GCB["LVT"]
|
|
22
|
+
|
|
23
|
+
_BREAKERS = frozenset({_CR, _LF, _CONTROL})
|
|
24
|
+
_AFTER_L = frozenset({_L, _V, _LV, _LVT})
|
|
25
|
+
_AFTER_LV_V = frozenset({_V, _T})
|
|
26
|
+
_AFTER_LVT_T = frozenset({_T})
|
|
27
|
+
_INCB_LINKER = 1
|
|
28
|
+
_INCB_CONSONANT = 2
|
|
29
|
+
_INCB_EXTEND = 3
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def grapheme_ends(s: str) -> Iterator[int]:
|
|
33
|
+
"""Yield the end index (exclusive) of every extended grapheme cluster in ``s``.
|
|
34
|
+
|
|
35
|
+
The last value yielded is ``len(s)``; nothing is yielded for the empty string.
|
|
36
|
+
"""
|
|
37
|
+
if not s:
|
|
38
|
+
return
|
|
39
|
+
code = prop(ord(s[0]))
|
|
40
|
+
prev = code & 0x0F
|
|
41
|
+
ri_run = 1 if prev == _RI else 0
|
|
42
|
+
incb = (code >> INCB_SHIFT) & 0x03
|
|
43
|
+
after_linker = incb == _INCB_LINKER
|
|
44
|
+
emoji = 1 if code & EXT_PICT_BIT else 0 # 1: pictograph Extend*, 2: then ZWJ
|
|
45
|
+
for index in range(1, len(s)):
|
|
46
|
+
code = prop(ord(s[index]))
|
|
47
|
+
cur = code & 0x0F
|
|
48
|
+
incb = (code >> INCB_SHIFT) & 0x03
|
|
49
|
+
pict = code & EXT_PICT_BIT
|
|
50
|
+
if not _joins(prev, cur, incb, pict, after_linker, emoji, ri_run):
|
|
51
|
+
yield index
|
|
52
|
+
after_linker = incb == _INCB_LINKER or (after_linker and incb == _INCB_EXTEND)
|
|
53
|
+
if pict or (emoji == 1 and cur == _EXTEND):
|
|
54
|
+
emoji = 1
|
|
55
|
+
elif emoji == 1 and cur == _ZWJ:
|
|
56
|
+
emoji = 2
|
|
57
|
+
else:
|
|
58
|
+
emoji = 0
|
|
59
|
+
ri_run = ri_run + 1 if cur == _RI else 0
|
|
60
|
+
prev = cur
|
|
61
|
+
yield len(s)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _joins(
|
|
65
|
+
prev: int, cur: int, incb: int, pict: int, after_linker: bool, emoji: int, ri_run: int
|
|
66
|
+
) -> bool:
|
|
67
|
+
"""Return True when there is no boundary between ``prev`` and ``cur``."""
|
|
68
|
+
if prev == _CR and cur == _LF: # GB3
|
|
69
|
+
return True
|
|
70
|
+
if prev in _BREAKERS or cur in _BREAKERS: # GB4, GB5
|
|
71
|
+
return False
|
|
72
|
+
if prev == _L and cur in _AFTER_L: # GB6
|
|
73
|
+
return True
|
|
74
|
+
if prev in (_LV, _V) and cur in _AFTER_LV_V: # GB7
|
|
75
|
+
return True
|
|
76
|
+
if prev in (_LVT, _T) and cur in _AFTER_LVT_T: # GB8
|
|
77
|
+
return True
|
|
78
|
+
if cur in (_EXTEND, _ZWJ, _SPACING) or prev == _PREPEND: # GB9, GB9a, GB9b
|
|
79
|
+
return True
|
|
80
|
+
if incb == _INCB_CONSONANT and after_linker: # GB9c
|
|
81
|
+
return True
|
|
82
|
+
if pict and prev == _ZWJ and emoji == 2: # GB11
|
|
83
|
+
return True
|
|
84
|
+
return prev == _RI and cur == _RI and ri_run % 2 == 1 # GB12, GB13
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def iter_graphemes(s: str) -> Iterator[str]:
|
|
88
|
+
"""Yield the extended grapheme clusters of ``s`` (UAX #29), left to right.
|
|
89
|
+
|
|
90
|
+
A grapheme cluster is what a reader perceives as one character: a base letter with
|
|
91
|
+
its combining marks, a Hangul syllable built from jamo, an emoji ZWJ sequence, a
|
|
92
|
+
flag made of two regional indicators, an Indic conjunct, or a CR LF pair. The
|
|
93
|
+
clusters concatenate back to ``s`` exactly.
|
|
94
|
+
|
|
95
|
+
>>> [len(g) for g in iter_graphemes("e\\u0301\\U0001f1ea\\U0001f1f8")]
|
|
96
|
+
[2, 2]
|
|
97
|
+
"""
|
|
98
|
+
start = 0
|
|
99
|
+
for end in grapheme_ends(s):
|
|
100
|
+
yield s[start:end]
|
|
101
|
+
start = end
|
cellstr/_ops.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Grapheme-safe slicing and padding by display cells.
|
|
2
|
+
|
|
3
|
+
Escape sequences are always kept in place so colour, style and hyperlinks survive the
|
|
4
|
+
operation, while only printable cells count toward a width.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from ._cells import Cell, check_tabsize, expand_tabs, iter_cells, total_width
|
|
10
|
+
from ._graphemes import grapheme_ends
|
|
11
|
+
from ._width import check_ambiguous, cluster_width, text_width
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def cell_slice(
|
|
15
|
+
text: str,
|
|
16
|
+
start: int = 0,
|
|
17
|
+
stop: int | None = None,
|
|
18
|
+
*,
|
|
19
|
+
ambiguous: int = 1,
|
|
20
|
+
tabsize: int | None = None,
|
|
21
|
+
) -> str:
|
|
22
|
+
"""Return the cells of ``text`` in the half-open column range ``[start, stop)``.
|
|
23
|
+
|
|
24
|
+
Grapheme clusters are never split: a letter keeps its accents, a flag or an emoji
|
|
25
|
+
ZWJ sequence stays whole, and a wide cluster is kept only when all its cells fall
|
|
26
|
+
inside the range. Every escape sequence is preserved wherever it sits, so styles and
|
|
27
|
+
OSC 8 hyperlinks still open and close. A zero-width cluster belongs to the column it
|
|
28
|
+
sits at, and one at the very end belongs to a non-empty range that reaches the end
|
|
29
|
+
from a column before it (for text with no width at all, to any range starting at 0
|
|
30
|
+
that is open-ended or non-empty). An empty range holds no printable text. With ``tabsize``
|
|
31
|
+
set, a tab becomes the spaces it covers inside the range, so the layout survives.
|
|
32
|
+
|
|
33
|
+
For plain text, ``cell_slice(t, 0, k) + cell_slice(t, k)`` equals ``t`` whenever
|
|
34
|
+
``k`` does not fall inside a wide cluster.
|
|
35
|
+
|
|
36
|
+
>>> cell_slice("\\x1b[31mhello\\x1b[0m", 0, 3) == "\\x1b[31mhel\\x1b[0m"
|
|
37
|
+
True
|
|
38
|
+
>>> cell_slice("a\\u4f60b", 0, 2), cell_slice("e\\u0301tude", 0, 1) == "e\\u0301"
|
|
39
|
+
('a', True)
|
|
40
|
+
"""
|
|
41
|
+
if start < 0 or (stop is not None and stop < 0):
|
|
42
|
+
raise ValueError("cell_slice does not support negative indices")
|
|
43
|
+
cells = list(iter_cells(text, ambiguous=ambiguous, tabsize=tabsize))
|
|
44
|
+
total = total_width(cells)
|
|
45
|
+
limit = total if stop is None else min(stop, total)
|
|
46
|
+
# A zero-width cluster at the very end: who gets it (see the docstring).
|
|
47
|
+
reach = (start < total) if total else (start == 0 and (stop is None or stop > 0))
|
|
48
|
+
tail = limit == total and reach
|
|
49
|
+
return "".join(_keep(cell, start, limit, total, tail, tabsize) for cell in cells)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _keep(cell: Cell, start: int, limit: int, total: int, tail: bool, tabsize: int | None) -> str:
|
|
53
|
+
if cell.escape:
|
|
54
|
+
return cell.text
|
|
55
|
+
if tabsize is not None and cell.text == "\t":
|
|
56
|
+
return " " * max(0, min(cell.column + cell.width, limit) - max(cell.column, start))
|
|
57
|
+
if cell.column < start:
|
|
58
|
+
return ""
|
|
59
|
+
if cell.width:
|
|
60
|
+
return "" if cell.column + cell.width > limit else cell.text
|
|
61
|
+
return cell.text if cell.column < limit or (tail and cell.column == total) else ""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def check_fill(fillchar: str, ambiguous: int) -> str:
|
|
65
|
+
"""Return ``fillchar`` after checking it is one grapheme cluster one cell wide."""
|
|
66
|
+
if (
|
|
67
|
+
not fillchar
|
|
68
|
+
or "\x1b" in fillchar
|
|
69
|
+
or next(grapheme_ends(fillchar)) != len(fillchar)
|
|
70
|
+
or cluster_width(fillchar, ambiguous) != 1
|
|
71
|
+
):
|
|
72
|
+
raise ValueError(f"fillchar must be a single one-cell character, not {fillchar!r}")
|
|
73
|
+
return fillchar
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _prepare(text: str, fillchar: str, ambiguous: int, tabsize: int | None) -> tuple[str, int]:
|
|
77
|
+
check_ambiguous(ambiguous)
|
|
78
|
+
check_fill(fillchar, ambiguous)
|
|
79
|
+
tabsize = check_tabsize(tabsize)
|
|
80
|
+
if tabsize is not None:
|
|
81
|
+
text = expand_tabs(text, tabsize, ambiguous=ambiguous)
|
|
82
|
+
return text, text_width(text, ambiguous)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def ljust(
|
|
86
|
+
text: str, width: int, fillchar: str = " ", *, ambiguous: int = 1, tabsize: int | None = None
|
|
87
|
+
) -> str:
|
|
88
|
+
"""Pad ``text`` on the right to ``width`` display cells.
|
|
89
|
+
|
|
90
|
+
Text already ``width`` cells or wider is returned unchanged. ``fillchar`` must be one
|
|
91
|
+
cell wide. With ``tabsize`` set, tabs are expanded first (see ``expand_tabs``).
|
|
92
|
+
|
|
93
|
+
>>> ljust("\\u4f60", 4) == "\\u4f60 "
|
|
94
|
+
True
|
|
95
|
+
"""
|
|
96
|
+
text, used = _prepare(text, fillchar, ambiguous, tabsize)
|
|
97
|
+
return text + fillchar * max(0, width - used)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def rjust(
|
|
101
|
+
text: str, width: int, fillchar: str = " ", *, ambiguous: int = 1, tabsize: int | None = None
|
|
102
|
+
) -> str:
|
|
103
|
+
"""Pad ``text`` on the left to ``width`` display cells.
|
|
104
|
+
|
|
105
|
+
>>> rjust("\\x1b[1m42\\x1b[0m", 5) == " \\x1b[1m42\\x1b[0m"
|
|
106
|
+
True
|
|
107
|
+
"""
|
|
108
|
+
text, used = _prepare(text, fillchar, ambiguous, tabsize)
|
|
109
|
+
return fillchar * max(0, width - used) + text
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def center(
|
|
113
|
+
text: str, width: int, fillchar: str = " ", *, ambiguous: int = 1, tabsize: int | None = None
|
|
114
|
+
) -> str:
|
|
115
|
+
"""Centre ``text`` within ``width`` display cells; an odd cell of padding goes right.
|
|
116
|
+
|
|
117
|
+
>>> center("ab", 5)
|
|
118
|
+
' ab '
|
|
119
|
+
"""
|
|
120
|
+
text, used = _prepare(text, fillchar, ambiguous, tabsize)
|
|
121
|
+
pad = max(0, width - used)
|
|
122
|
+
left = pad // 2
|
|
123
|
+
return fillchar * left + text + fillchar * (pad - left)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def splice(text: str, head: int, rstart: int, insert: str, ambiguous: int) -> str:
|
|
127
|
+
"""Keep the clusters in ``[0, head)`` and ``[rstart, end)``, putting ``insert`` between.
|
|
128
|
+
|
|
129
|
+
Every escape sequence is kept exactly once, in place, and ``insert`` goes where the
|
|
130
|
+
first dropped cluster was.
|
|
131
|
+
"""
|
|
132
|
+
cells = list(iter_cells(text, ambiguous=ambiguous))
|
|
133
|
+
total = total_width(cells)
|
|
134
|
+
out: list[str] = []
|
|
135
|
+
pending = True
|
|
136
|
+
for cell in cells:
|
|
137
|
+
end = cell.column + cell.width
|
|
138
|
+
left = end <= head if cell.width else cell.column < head
|
|
139
|
+
right = rstart <= cell.column and end <= total
|
|
140
|
+
if not (cell.escape or left) and pending:
|
|
141
|
+
out.append(insert)
|
|
142
|
+
pending = False
|
|
143
|
+
if cell.escape or left or right:
|
|
144
|
+
out.append(cell.text)
|
|
145
|
+
if pending:
|
|
146
|
+
out.append(insert)
|
|
147
|
+
return "".join(out)
|