cjk-semantic-split 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chinese_decompose/__init__.py +2 -0
- chinese_decompose/__main__.py +134 -0
- chinese_decompose/api.py +135 -0
- chinese_decompose/data/decomp_l1.json +12 -0
- chinese_decompose/data/decomp_l2.json +1 -0
- chinese_decompose/data/decomp_l3.json +5 -0
- chinese_decompose/data/decomp_patches.json +1 -0
- chinese_decompose/data/primitives.json +21 -0
- chinese_decompose/dsl.py +102 -0
- chinese_decompose/errors.py +26 -0
- chinese_decompose/loader.py +132 -0
- chinese_decompose/primitives.py +41 -0
- chinese_decompose/split.py +52 -0
- chinese_decompose/structures.py +127 -0
- cjk_semantic_split-0.1.0.dist-info/METADATA +102 -0
- cjk_semantic_split-0.1.0.dist-info/RECORD +18 -0
- cjk_semantic_split-0.1.0.dist-info/WHEEL +4 -0
- cjk_semantic_split-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""CLI entry point: python -m chinese_decompose."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from chinese_decompose import __version__
|
|
10
|
+
from chinese_decompose.api import decompose
|
|
11
|
+
from chinese_decompose.errors import ChineseDecomposeError, UnknownCharacterError
|
|
12
|
+
from chinese_decompose.loader import make_loader
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
prog="chinese_decompose",
|
|
18
|
+
description="Decompose CJK characters into atomic primitives with spatial positions.",
|
|
19
|
+
)
|
|
20
|
+
parser.add_argument(
|
|
21
|
+
"chars",
|
|
22
|
+
nargs="*",
|
|
23
|
+
help="One or more CJK characters to decompose. If empty, read from stdin.",
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"--format",
|
|
27
|
+
choices=["dsl", "json", "tree"],
|
|
28
|
+
default="json",
|
|
29
|
+
help="Output format (default: json).",
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument(
|
|
32
|
+
"--no-strict",
|
|
33
|
+
action="store_true",
|
|
34
|
+
help="Emit ? markers for unknown characters instead of raising.",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--primitives",
|
|
38
|
+
type=Path,
|
|
39
|
+
help="Path to a JSON file overriding the primitive set.",
|
|
40
|
+
)
|
|
41
|
+
parser.add_argument(
|
|
42
|
+
"--self-test",
|
|
43
|
+
action="store_true",
|
|
44
|
+
help="Run a self-test reporting tier coverage.",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--version",
|
|
48
|
+
action="version",
|
|
49
|
+
version=f"chinese_decompose {__version__}",
|
|
50
|
+
)
|
|
51
|
+
return parser
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _read_stdin() -> str:
|
|
55
|
+
return sys.stdin.read().strip()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _format_dsl(text: str, strict: bool) -> str:
|
|
59
|
+
return decompose(text, strict=strict)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _format_json(text: str, strict: bool) -> str:
|
|
63
|
+
trees = decompose(text, as_tree=True, strict=strict)
|
|
64
|
+
out = []
|
|
65
|
+
for tree in trees:
|
|
66
|
+
out.append({
|
|
67
|
+
"char": tree.char,
|
|
68
|
+
"atoms": list(tree.atoms),
|
|
69
|
+
"positions": list(tree.positions),
|
|
70
|
+
"dsl": tree.to_dsl(),
|
|
71
|
+
})
|
|
72
|
+
return json.dumps(out, ensure_ascii=False, indent=2)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _format_tree(text: str, strict: bool) -> str:
|
|
76
|
+
trees = decompose(text, as_tree=True, strict=strict)
|
|
77
|
+
return "\n".join(repr(t) for t in trees)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _self_test() -> str:
|
|
81
|
+
loader = make_loader()
|
|
82
|
+
total = len(loader)
|
|
83
|
+
return (
|
|
84
|
+
f"chinese_decompose self-test\n"
|
|
85
|
+
f" version: {__version__}\n"
|
|
86
|
+
f" characters indexed: {total}\n"
|
|
87
|
+
f" tiers: 4 (l1, l2, l3, patches)\n"
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main(argv: list[str] | None = None) -> int:
|
|
92
|
+
parser = _build_parser()
|
|
93
|
+
args = parser.parse_args(argv)
|
|
94
|
+
|
|
95
|
+
if args.primitives is not None:
|
|
96
|
+
data = json.loads(args.primitives.read_text(encoding="utf-8"))
|
|
97
|
+
from chinese_decompose.primitives import set_primitives
|
|
98
|
+
if isinstance(data, list):
|
|
99
|
+
set_primitives(data)
|
|
100
|
+
elif isinstance(data, dict) and "primitives" in data:
|
|
101
|
+
set_primitives(data["primitives"])
|
|
102
|
+
|
|
103
|
+
if args.self_test:
|
|
104
|
+
sys.stdout.write(_self_test())
|
|
105
|
+
return 0
|
|
106
|
+
|
|
107
|
+
text = "".join(args.chars) if args.chars else _read_stdin()
|
|
108
|
+
if not text:
|
|
109
|
+
parser.print_help()
|
|
110
|
+
return 1
|
|
111
|
+
|
|
112
|
+
strict = not args.no_strict
|
|
113
|
+
try:
|
|
114
|
+
if args.format == "dsl":
|
|
115
|
+
output = _format_dsl(text, strict)
|
|
116
|
+
elif args.format == "json":
|
|
117
|
+
output = _format_json(text, strict)
|
|
118
|
+
elif args.format == "tree":
|
|
119
|
+
output = _format_tree(text, strict)
|
|
120
|
+
else:
|
|
121
|
+
output = ""
|
|
122
|
+
except UnknownCharacterError as e:
|
|
123
|
+
sys.stderr.write(f"Error: UnknownCharacterError: {e}\n")
|
|
124
|
+
return 2
|
|
125
|
+
except ChineseDecomposeError as e:
|
|
126
|
+
sys.stderr.write(f"Error: {type(e).__name__}: {e}\n")
|
|
127
|
+
return 1
|
|
128
|
+
|
|
129
|
+
sys.stdout.write(output + "\n")
|
|
130
|
+
return 0
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__":
|
|
134
|
+
sys.exit(main())
|
chinese_decompose/api.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Decomposition tree and DSL serialization helpers."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Iterable, Union
|
|
6
|
+
|
|
7
|
+
from chinese_decompose.errors import UnknownCharacterError
|
|
8
|
+
from chinese_decompose.loader import make_loader as _make_loader
|
|
9
|
+
from chinese_decompose.split import split_one
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True)
|
|
13
|
+
class DecompositionTree:
|
|
14
|
+
"""Immutable decomposition result for a single character.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
char: The original character (str of length 1).
|
|
18
|
+
children: Tuple of sub-trees at the next level.
|
|
19
|
+
position: The character's own 2-digit position code (set by parent).
|
|
20
|
+
|
|
21
|
+
Derived properties:
|
|
22
|
+
atoms: Flat list of leaf characters (primitives) in DFS order.
|
|
23
|
+
positions: Flat list of leaf positions corresponding to atoms.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
char: str
|
|
27
|
+
children: tuple["DecompositionTree", ...] = ()
|
|
28
|
+
position: int | None = None
|
|
29
|
+
|
|
30
|
+
def __post_init__(self) -> None:
|
|
31
|
+
if not isinstance(self.char, str) or len(self.char) != 1:
|
|
32
|
+
raise ValueError(
|
|
33
|
+
f"char must be a single-character string, got {self.char!r}"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def atoms(self) -> tuple[str, ...]:
|
|
38
|
+
"""Leaf (primitive) characters in DFS decomposition order."""
|
|
39
|
+
if not self.children:
|
|
40
|
+
return (self.char,)
|
|
41
|
+
out: list[str] = []
|
|
42
|
+
for child in self.children:
|
|
43
|
+
out.extend(child.atoms)
|
|
44
|
+
return tuple(out)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def positions(self) -> tuple[int, ...]:
|
|
48
|
+
"""Leaf positions corresponding to atoms, in DFS order."""
|
|
49
|
+
if not self.children:
|
|
50
|
+
return (self.position,) if self.position is not None else ()
|
|
51
|
+
out: list[int] = []
|
|
52
|
+
for child in self.children:
|
|
53
|
+
out.extend(child.positions)
|
|
54
|
+
return tuple(out)
|
|
55
|
+
|
|
56
|
+
def to_dsl(self) -> str:
|
|
57
|
+
"""Serialize to DSL format per spec §5.1.
|
|
58
|
+
|
|
59
|
+
Staggered format: each segment's bracket carries the NEXT leaf's
|
|
60
|
+
position; the final leaf is bare. A single space is inserted between
|
|
61
|
+
every character (root or leaf) and its following bracketed position,
|
|
62
|
+
e.g. `踩 [30]:足 [60]:爪 [80]:木`. Primitives emit just the char.
|
|
63
|
+
"""
|
|
64
|
+
if not self.children:
|
|
65
|
+
return self.char
|
|
66
|
+
|
|
67
|
+
leaves = self.atoms
|
|
68
|
+
leaf_positions = self.positions
|
|
69
|
+
|
|
70
|
+
if not leaf_positions:
|
|
71
|
+
return self.char + ":" + ":".join(leaves)
|
|
72
|
+
|
|
73
|
+
parts: list[str] = [f"{self.char} [{leaf_positions[0]}]"]
|
|
74
|
+
for i in range(len(leaves) - 1):
|
|
75
|
+
parts.append(f"{leaves[i]} [{leaf_positions[i + 1]}]")
|
|
76
|
+
parts.append(leaves[-1])
|
|
77
|
+
return ":".join(parts)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def decompose(
|
|
81
|
+
text: str,
|
|
82
|
+
as_tree: bool = False,
|
|
83
|
+
strict: bool = True,
|
|
84
|
+
loader=None,
|
|
85
|
+
) -> Union[str, list[DecompositionTree]]:
|
|
86
|
+
"""Decompose each character in `text` and return DSL string or list of trees.
|
|
87
|
+
|
|
88
|
+
Args:
|
|
89
|
+
text: String of one or more CJK characters.
|
|
90
|
+
as_tree: If True, return list of DecompositionTree (one per char).
|
|
91
|
+
If False (default), return the flat DSL string.
|
|
92
|
+
strict: If True (default), raise UnknownCharacterError on missing data.
|
|
93
|
+
If False, emit `?[?]:?` markers.
|
|
94
|
+
loader: Optional Loader instance. Defaults to singleton via make_loader().
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
If as_tree=False: a DSL string. Multi-char input joined by `|`.
|
|
98
|
+
If as_tree=True: a list of DecompositionTree (one per char, in input order).
|
|
99
|
+
"""
|
|
100
|
+
if loader is None:
|
|
101
|
+
loader = _make_loader()
|
|
102
|
+
|
|
103
|
+
trees: list[DecompositionTree] = []
|
|
104
|
+
for char in text:
|
|
105
|
+
flat = split_one(char, loader)
|
|
106
|
+
if flat is None:
|
|
107
|
+
if strict:
|
|
108
|
+
raise UnknownCharacterError(char)
|
|
109
|
+
trees.append(_marker_tree())
|
|
110
|
+
continue
|
|
111
|
+
|
|
112
|
+
if not flat:
|
|
113
|
+
# Primitive leaf
|
|
114
|
+
trees.append(DecompositionTree(char=char, children=()))
|
|
115
|
+
else:
|
|
116
|
+
children = tuple(
|
|
117
|
+
DecompositionTree(char=atom, children=(), position=pos)
|
|
118
|
+
for atom, pos in flat
|
|
119
|
+
)
|
|
120
|
+
trees.append(DecompositionTree(char=char, children=children))
|
|
121
|
+
|
|
122
|
+
if as_tree:
|
|
123
|
+
return trees
|
|
124
|
+
return "|".join(t.to_dsl() for t in trees)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _marker_tree() -> DecompositionTree:
|
|
128
|
+
"""Marker tree for unknown characters in non-strict mode."""
|
|
129
|
+
return DecompositionTree(
|
|
130
|
+
char="?",
|
|
131
|
+
children=(DecompositionTree(char="?", children=(), position=0),),
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
__all__ = ["DecompositionTree", "decompose"]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_meta": {"tier": 1, "source": "manual-fixture"},
|
|
3
|
+
"好": [["女", 30], ["子", 40]],
|
|
4
|
+
"森": [["木", 10], ["木", 20], ["木", 40]],
|
|
5
|
+
"囚": [["囗", 90], ["人", 0]],
|
|
6
|
+
"回": [["囗", 90], ["口", 0]],
|
|
7
|
+
"男": [["田", 10], ["力", 20]],
|
|
8
|
+
"休": [["亻", 30], ["木", 40]],
|
|
9
|
+
"明": [["日", 30], ["月", 40]],
|
|
10
|
+
"晶": [["日", 10], ["日", 20], ["日", 40]],
|
|
11
|
+
"品": [["口", 10], ["口", 20], ["口", 40]]
|
|
12
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"_meta": {"tier": 2, "source": "manual-fixture"}}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"_meta": {"tier": 4, "source": "manual-fixture"}}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"primitives": [
|
|
3
|
+
"木","火","土","金","水","人","口","手","心","日","月","目","耳","鼻","舌","身",
|
|
4
|
+
"爪","戈","矢","石","一","乙","二","三","四","五","六","七","八","九","十","丁",
|
|
5
|
+
"卜","厂","寸","干","弓","牛","犬","玉","王","中","山","川","田","禾","竹","米",
|
|
6
|
+
"糸","网","羊","马","鸟","鱼","虫","贝","车","舟","门","阝","亻","刂","冫","氵",
|
|
7
|
+
"犭","忄","宀","艹","辶","阝","饣","亠","匚","凵","冂","卩","廴","彳","攴","疒",
|
|
8
|
+
"癶","衤","礻","纟","讠","钅","页","风","云","雨","雪","雷","女","子","父","母",
|
|
9
|
+
"兄","弟","姐","妹","己","已","巳","工","土","才","寸","巾","干","幺","广","廴",
|
|
10
|
+
"彐","彡","扌","支","攴","文","斗","斤","方","无","日","曰","月","木","欠","止",
|
|
11
|
+
"歹","殳","毋","比","毛","氏","气","水","火","爪","父","爻","爿","片","牙","牛",
|
|
12
|
+
"犬","玄","玉","瓜","瓦","甘","生","用","田","疋","疒","癶","白","皮","皿","目",
|
|
13
|
+
"矛","矢","石","示","禸","禾","穴","立","竹","米","糸","缶","网","羊","羽","老",
|
|
14
|
+
"而","耒","耳","聿","肉","臣","自","至","臼","舌","舛","舟","艮","色","艸","虍",
|
|
15
|
+
"虫","血","行","衣","襾","覀","角","言","谷","豆","豕","豸","貝","赤","走","足",
|
|
16
|
+
"身","車","辛","辰","辵","邑","酉","釆","里","金","長","門","阜","隶","隹","雨",
|
|
17
|
+
"靑","非","面","革","韋","音","頁","風","飛","食","首","香","馬","骨","高","髟",
|
|
18
|
+
"鬥","鬯","鬲","鬼","魚","鳥","鹵","鹿","麥","麻","黃","黍","黑","黹","黽","鼎",
|
|
19
|
+
"鼓","鼠","鼻","齊","亀","龍","龜","龠","囗"
|
|
20
|
+
]
|
|
21
|
+
}
|
chinese_decompose/dsl.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""DSL serializer and parser per spec §5."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from typing import Iterable, Union
|
|
6
|
+
|
|
7
|
+
from chinese_decompose.api import DecompositionTree
|
|
8
|
+
from chinese_decompose.errors import MalformedDSLParseError
|
|
9
|
+
from chinese_decompose.structures import VALID_POSITIONS
|
|
10
|
+
|
|
11
|
+
# Match a single char (CJK Unified + ASCII fallback) with optional [pos].
|
|
12
|
+
# Allows an optional space between char and [pos] (user-facing format).
|
|
13
|
+
_SEGMENT_PATTERN = re.compile(r"^([一-鿿 -〿?])\s*(?:\[(-?\d+)\])?$")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def serialize_tree(tree: DecompositionTree) -> str:
|
|
17
|
+
"""Serialize a single DecompositionTree to DSL.
|
|
18
|
+
|
|
19
|
+
Delegates to DecompositionTree.to_dsl() — same staggered format
|
|
20
|
+
(each segment's bracket is the NEXT leaf's position; last leaf is bare)
|
|
21
|
+
with spaces between chars and brackets.
|
|
22
|
+
"""
|
|
23
|
+
return tree.to_dsl()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def serialize_many(trees: Iterable[DecompositionTree]) -> str:
|
|
27
|
+
"""Serialize multiple trees joined by `|` (per spec §5.3)."""
|
|
28
|
+
return "|".join(serialize_tree(t) for t in trees)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _parse_position(raw: str | None) -> int | None:
|
|
32
|
+
if raw is None:
|
|
33
|
+
return None
|
|
34
|
+
try:
|
|
35
|
+
code = int(raw)
|
|
36
|
+
except ValueError as e:
|
|
37
|
+
raise MalformedDSLParseError(f"Position code not an integer: {raw!r}") from e
|
|
38
|
+
if code not in VALID_POSITIONS:
|
|
39
|
+
raise MalformedDSLParseError(f"Position code {code!r} not in valid set")
|
|
40
|
+
return code
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def parse_dsl(dsl_str: str) -> DecompositionTree:
|
|
44
|
+
"""Parse a DSL string into a DecompositionTree.
|
|
45
|
+
|
|
46
|
+
Format: `<root>[<pos>]:<segment>:<segment>:...:<lastLeaf>`.
|
|
47
|
+
|
|
48
|
+
The last leaf is bare (no brackets). Earlier segments may also lack
|
|
49
|
+
brackets (treated as position=None). Parser is 1-level — produces a flat
|
|
50
|
+
tree where children are leaves (intermediate non-leaf nodes are not
|
|
51
|
+
reconstructed; full round-trip would need richer nesting logic, deferred
|
|
52
|
+
to a future task).
|
|
53
|
+
|
|
54
|
+
Raises:
|
|
55
|
+
MalformedDSLParseError: on empty input, missing/invalid chars,
|
|
56
|
+
non-numeric or out-of-set position codes, or empty segments.
|
|
57
|
+
"""
|
|
58
|
+
if not dsl_str:
|
|
59
|
+
raise MalformedDSLParseError("Empty DSL string")
|
|
60
|
+
|
|
61
|
+
raw_segments = dsl_str.split(":")
|
|
62
|
+
if not raw_segments or not raw_segments[0]:
|
|
63
|
+
raise MalformedDSLParseError("Missing root segment")
|
|
64
|
+
|
|
65
|
+
root_match = _SEGMENT_PATTERN.match(raw_segments[0])
|
|
66
|
+
if not root_match:
|
|
67
|
+
raise MalformedDSLParseError(f"Invalid root segment: {raw_segments[0]!r}")
|
|
68
|
+
root_char = root_match.group(1)
|
|
69
|
+
root_position = _parse_position(root_match.group(2))
|
|
70
|
+
|
|
71
|
+
# Detect malformed (missing/extra colons). Bare segments are allowed only
|
|
72
|
+
# for the very last one.
|
|
73
|
+
children: list[DecompositionTree] = []
|
|
74
|
+
for idx, seg in enumerate(raw_segments[1:], start=1):
|
|
75
|
+
last = idx == len(raw_segments) - 1
|
|
76
|
+
if not seg:
|
|
77
|
+
if last and idx == 1:
|
|
78
|
+
# Just `好[30]:` — trailing colon, treat as 1-leaf tree
|
|
79
|
+
break
|
|
80
|
+
raise MalformedDSLParseError(f"Empty segment at index {idx}")
|
|
81
|
+
m = _SEGMENT_PATTERN.match(seg)
|
|
82
|
+
if not m:
|
|
83
|
+
raise MalformedDSLParseError(f"Invalid segment: {seg!r}")
|
|
84
|
+
child_char = m.group(1)
|
|
85
|
+
child_pos_raw = m.group(2)
|
|
86
|
+
if child_pos_raw is None and not last:
|
|
87
|
+
raise MalformedDSLParseError(f"Non-final segment missing position: {seg!r}")
|
|
88
|
+
child_position = _parse_position(child_pos_raw)
|
|
89
|
+
children.append(
|
|
90
|
+
DecompositionTree(char=child_char, children=(), position=child_position)
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
return DecompositionTree(
|
|
94
|
+
char=root_char,
|
|
95
|
+
children=tuple(children),
|
|
96
|
+
position=root_position,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_dsl_multi(dsl_str: str) -> list[DecompositionTree]:
|
|
101
|
+
"""Parse a multi-character DSL (joined by `|`)."""
|
|
102
|
+
return [parse_dsl(part) for part in dsl_str.split("|") if part]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Exception hierarchy for chinese_decompose."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ChineseDecomposeError(Exception):
|
|
6
|
+
"""Base class for all chinese_decompose errors."""
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class UnknownCharacterError(ChineseDecomposeError):
|
|
10
|
+
"""Raised when a character has no entry across all data tiers."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, char: str):
|
|
13
|
+
self.char = char
|
|
14
|
+
super().__init__(f"No decomposition data for character: {char!r}")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class MalformedDSLParseError(ChineseDecomposeError):
|
|
18
|
+
"""Raised when DSL parsing fails (bad position, malformed segment)."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class InvalidPositionCodeError(ChineseDecomposeError):
|
|
22
|
+
"""Raised when a data file or input contains a code outside the valid 2-digit set."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class CompositionError(ChineseDecomposeError):
|
|
26
|
+
"""Raised by compose(parent, child) when the combination is geometrically invalid."""
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Multi-tier decomposition data loader with last-wins resolution."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from chinese_decompose.errors import InvalidPositionCodeError
|
|
9
|
+
from chinese_decompose.structures import VALID_POSITIONS
|
|
10
|
+
|
|
11
|
+
_TIER_FILES: dict[str, Path | None] = {}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _resolve_default_paths() -> tuple[Path, Path, Path, Path]:
|
|
15
|
+
data = Path(__file__).parent / "data"
|
|
16
|
+
return (
|
|
17
|
+
data / "decomp_l1.json",
|
|
18
|
+
data / "decomp_l2.json",
|
|
19
|
+
data / "decomp_l3.json",
|
|
20
|
+
data / "decomp_patches.json",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _validate_entry(char: str, entries: Any) -> list[tuple[str, int]]:
|
|
25
|
+
if not isinstance(entries, list):
|
|
26
|
+
raise InvalidPositionCodeError(
|
|
27
|
+
f"Entry for {char!r} must be a list of [char, code] pairs"
|
|
28
|
+
)
|
|
29
|
+
result = []
|
|
30
|
+
for entry in entries:
|
|
31
|
+
if not isinstance(entry, list) or len(entry) != 2:
|
|
32
|
+
raise InvalidPositionCodeError(
|
|
33
|
+
f"Entry for {char!r} malformed: {entry!r}"
|
|
34
|
+
)
|
|
35
|
+
sub_char, code = entry
|
|
36
|
+
if not isinstance(sub_char, str) or len(sub_char) != 1:
|
|
37
|
+
raise InvalidPositionCodeError(
|
|
38
|
+
f"Sub-component for {char!r} must be a single char, got {sub_char!r}"
|
|
39
|
+
)
|
|
40
|
+
if not isinstance(code, int) or code not in VALID_POSITIONS:
|
|
41
|
+
raise InvalidPositionCodeError(
|
|
42
|
+
f"Position code for {char!r}: {code!r} not in valid set"
|
|
43
|
+
)
|
|
44
|
+
result.append((sub_char, code))
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _load_json(path: Path) -> dict[str, list[tuple[str, int]]]:
|
|
49
|
+
if not path.exists():
|
|
50
|
+
return {}
|
|
51
|
+
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
52
|
+
# Strip metadata
|
|
53
|
+
raw.pop("_meta", None)
|
|
54
|
+
out: dict[str, list[tuple[str, int]]] = {}
|
|
55
|
+
for char, entries in raw.items():
|
|
56
|
+
out[char] = _validate_entry(char, entries)
|
|
57
|
+
return out
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Loader:
|
|
61
|
+
"""Multi-tier decomposition loader.
|
|
62
|
+
|
|
63
|
+
Resolution order: patches > l3 > l2 > l1 (last-wins).
|
|
64
|
+
Caches results in-process; call clear_cache() to reset.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
_instance: "Loader | None" = None
|
|
68
|
+
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
l1_path: Path,
|
|
72
|
+
l2_path: Path,
|
|
73
|
+
l3_path: Path,
|
|
74
|
+
patches_path: Path,
|
|
75
|
+
):
|
|
76
|
+
self._paths = (l1_path, l2_path, l3_path, patches_path)
|
|
77
|
+
self._cache: dict[str, list[tuple[str, int]] | None] = {}
|
|
78
|
+
# Pre-load all data; raise on invalid codes
|
|
79
|
+
l1 = _load_json(l1_path)
|
|
80
|
+
l2 = _load_json(l2_path)
|
|
81
|
+
l3 = _load_json(l3_path)
|
|
82
|
+
patches = _load_json(patches_path)
|
|
83
|
+
self._merged = {}
|
|
84
|
+
for char in l1:
|
|
85
|
+
self._merged[char] = l1[char]
|
|
86
|
+
for char in l2:
|
|
87
|
+
self._merged[char] = l2[char]
|
|
88
|
+
for char in l3:
|
|
89
|
+
self._merged[char] = l3[char]
|
|
90
|
+
for char in patches:
|
|
91
|
+
self._merged[char] = patches[char]
|
|
92
|
+
|
|
93
|
+
def get(self, char: str) -> list[tuple[str, int]] | None:
|
|
94
|
+
"""Return one-level decomposition for `char`, or None if unknown."""
|
|
95
|
+
if char in self._cache:
|
|
96
|
+
return self._cache[char]
|
|
97
|
+
result = self._merged.get(char)
|
|
98
|
+
self._cache[char] = result
|
|
99
|
+
return result
|
|
100
|
+
|
|
101
|
+
def __len__(self) -> int:
|
|
102
|
+
return len(self._merged)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def make_loader(
|
|
106
|
+
l1_path: Path | None = None,
|
|
107
|
+
l2_path: Path | None = None,
|
|
108
|
+
l3_path: Path | None = None,
|
|
109
|
+
patches_path: Path | None = None,
|
|
110
|
+
) -> Loader:
|
|
111
|
+
"""Construct (or return cached singleton) Loader instance."""
|
|
112
|
+
if (
|
|
113
|
+
l1_path is None
|
|
114
|
+
and l2_path is None
|
|
115
|
+
and l3_path is None
|
|
116
|
+
and patches_path is None
|
|
117
|
+
):
|
|
118
|
+
p1, p2, p3, pp = _resolve_default_paths()
|
|
119
|
+
else:
|
|
120
|
+
d1, d2, d3, dp = _resolve_default_paths()
|
|
121
|
+
p1 = l1_path or d1
|
|
122
|
+
p2 = l2_path or d2
|
|
123
|
+
p3 = l3_path or d3
|
|
124
|
+
pp = patches_path or dp
|
|
125
|
+
Loader._instance = Loader(p1, p2, p3, pp)
|
|
126
|
+
return Loader._instance
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def clear_cache() -> None:
|
|
130
|
+
"""Reset loader cache; next get() rebuilds from JSON."""
|
|
131
|
+
if Loader._instance is not None:
|
|
132
|
+
Loader._instance._cache.clear()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Primitive set: lookup and process-local override."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Iterable
|
|
7
|
+
|
|
8
|
+
_DATA_PATH = Path(__file__).parent / "data" / "primitives.json"
|
|
9
|
+
_DEFAULT: frozenset[str] = frozenset(
|
|
10
|
+
json.loads(_DATA_PATH.read_text(encoding="utf-8"))["primitives"]
|
|
11
|
+
)
|
|
12
|
+
_OVERRIDE: frozenset[str] | None = None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_primitives() -> frozenset[str]:
|
|
16
|
+
"""Return the currently-active primitive set (override or default)."""
|
|
17
|
+
return _OVERRIDE if _OVERRIDE is not None else _DEFAULT
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_primitives(iterable: Iterable[str]) -> None:
|
|
21
|
+
"""Replace the primitive set for the current process.
|
|
22
|
+
|
|
23
|
+
Pass-through: useful for testing with custom primitive subsets.
|
|
24
|
+
Reset by passing None or by re-importing the module. Does NOT persist.
|
|
25
|
+
"""
|
|
26
|
+
global _OVERRIDE
|
|
27
|
+
items = list(iterable)
|
|
28
|
+
if not all(isinstance(s, str) and len(s) == 1 for s in items):
|
|
29
|
+
raise ValueError("primitives must be single-character strings")
|
|
30
|
+
_OVERRIDE = frozenset(items)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_primitive(char: str) -> bool:
|
|
34
|
+
"""Return True if `char` is in the active primitive set."""
|
|
35
|
+
return char in get_primitives()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def reset_primitives() -> None:
|
|
39
|
+
"""Clear any process-local override, restoring bundled defaults."""
|
|
40
|
+
global _OVERRIDE
|
|
41
|
+
_OVERRIDE = None
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Recursive decomposition walker with position composition."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from chinese_decompose.primitives import is_primitive
|
|
7
|
+
from chinese_decompose.structures import compose
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from chinese_decompose.loader import Loader
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def split_one(char: str, loader: "Loader") -> list[tuple[str, int]] | None:
|
|
14
|
+
"""Recursively decompose `char` into (atom_char, final_position) pairs.
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
- [] if `char` is a primitive (no further decomposition).
|
|
18
|
+
- list of (atom_char, position) pairs after full recursive expansion.
|
|
19
|
+
- None if `char` has no entry in the loader.
|
|
20
|
+
|
|
21
|
+
The recursion chains position composition: for each sub-character at its
|
|
22
|
+
declared local position, compose with the parent's position to derive the
|
|
23
|
+
final 9-grid position.
|
|
24
|
+
"""
|
|
25
|
+
return _split_recursive(char, parent_position=0, loader=loader)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _split_recursive(
|
|
29
|
+
char: str, parent_position: int, loader: "Loader"
|
|
30
|
+
) -> list[tuple[str, int]] | None:
|
|
31
|
+
"""Recursively walk the decomposition tree, composing positions."""
|
|
32
|
+
if is_primitive(char):
|
|
33
|
+
# Primitives terminate; no expansion. Empty list signals leaf.
|
|
34
|
+
return []
|
|
35
|
+
|
|
36
|
+
decomp = loader.get(char)
|
|
37
|
+
if decomp is None:
|
|
38
|
+
return None # unknown
|
|
39
|
+
|
|
40
|
+
out: list[tuple[str, int]] = []
|
|
41
|
+
for sub_char, local_pos in decomp:
|
|
42
|
+
final_pos = compose(parent_position, local_pos)
|
|
43
|
+
sub_result = _split_recursive(sub_char, parent_position=final_pos, loader=loader)
|
|
44
|
+
if sub_result is None:
|
|
45
|
+
# Sub-char unknown at this level; bubble up
|
|
46
|
+
return None
|
|
47
|
+
if sub_result:
|
|
48
|
+
out.extend(sub_result)
|
|
49
|
+
else:
|
|
50
|
+
# Primitive sub-character: emit just the char with final position
|
|
51
|
+
out.append((sub_char, final_pos))
|
|
52
|
+
return out
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Position code constants for the 9-grid spatial encoding.
|
|
2
|
+
|
|
3
|
+
Codes are 2-digit numbers (spec §2.1): digit A is the spatial zone, digit B
|
|
4
|
+
is the sub-zone within the parent zone (0 means atomic slot, non-zero for
|
|
5
|
+
specific sub-strips or surround variants).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from chinese_decompose.errors import CompositionError
|
|
10
|
+
|
|
11
|
+
# Atomic single-slot positions
|
|
12
|
+
P_CENTER = 0
|
|
13
|
+
P_TOP = 10
|
|
14
|
+
P_BOTTOM = 20
|
|
15
|
+
P_LEFT = 30
|
|
16
|
+
P_RIGHT = 40
|
|
17
|
+
P_TOP_LEFT = 50
|
|
18
|
+
P_TOP_RIGHT = 60
|
|
19
|
+
P_BOTTOM_LEFT = 70
|
|
20
|
+
P_BOTTOM_RIGHT = 80
|
|
21
|
+
|
|
22
|
+
# 4-cell layout strip positions
|
|
23
|
+
P_STRIP_VERT_RIGHT = 14 # inner-right-edge of upper zone (e.g. 目's right strip)
|
|
24
|
+
P_STRIP_VERT_LEFT = 16 # inner-left-edge of lower zone
|
|
25
|
+
P_STRIP_HORIZ_RIGHT = 34 # inner-right-edge of left zone (e.g. 罒's right strip)
|
|
26
|
+
P_STRIP_HORIZ_LEFT = 36 # inner-left-edge of right zone
|
|
27
|
+
|
|
28
|
+
# Surround envelopes (90-99)
|
|
29
|
+
P_SURROUND_FULL = 90
|
|
30
|
+
P_SURROUND_BELOW = 91
|
|
31
|
+
P_SURROUND_LEFT = 92
|
|
32
|
+
P_SURROUND_ABOVE = 93
|
|
33
|
+
P_SURROUND_RIGHT = 94
|
|
34
|
+
P_SURROUND_UL = 95
|
|
35
|
+
P_SURROUND_LR = 96
|
|
36
|
+
P_SURROUND_UR = 97
|
|
37
|
+
P_SURROUND_LL_LO = 98 # lower-bottom extension (semantics TBD per spec)
|
|
38
|
+
P_SURROUND_LL = 99
|
|
39
|
+
|
|
40
|
+
VALID_POSITIONS: frozenset[int] = frozenset({
|
|
41
|
+
P_CENTER, P_TOP, P_BOTTOM, P_LEFT, P_RIGHT,
|
|
42
|
+
P_TOP_LEFT, P_TOP_RIGHT, P_BOTTOM_LEFT, P_BOTTOM_RIGHT,
|
|
43
|
+
P_STRIP_VERT_RIGHT, P_STRIP_VERT_LEFT,
|
|
44
|
+
P_STRIP_HORIZ_RIGHT, P_STRIP_HORIZ_LEFT,
|
|
45
|
+
P_SURROUND_FULL, P_SURROUND_BELOW, P_SURROUND_LEFT,
|
|
46
|
+
P_SURROUND_ABOVE, P_SURROUND_RIGHT, P_SURROUND_UL,
|
|
47
|
+
P_SURROUND_LR, P_SURROUND_UR, P_SURROUND_LL,
|
|
48
|
+
# NOTE: P_SURROUND_LL_LO (98) intentionally excluded — semantics TBD per spec §2.1
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# Composition table (spec §2.3): (parent, child) -> composed position.
|
|
53
|
+
# Missing keys are geometrically invalid and raise CompositionError at runtime.
|
|
54
|
+
COMPOSE_TABLE: dict[tuple[int, int], int] = {
|
|
55
|
+
# 00 (center) row — center passes any valid position through unchanged.
|
|
56
|
+
(0, 0): 0, (0, 10): 10, (0, 20): 20, (0, 30): 30, (0, 40): 40,
|
|
57
|
+
(0, 50): 50, (0, 60): 60, (0, 70): 70, (0, 80): 80,
|
|
58
|
+
(0, 14): 14, (0, 16): 16, (0, 34): 34, (0, 36): 36,
|
|
59
|
+
(0, 90): 90, (0, 91): 91, (0, 92): 92, (0, 93): 93, (0, 94): 94,
|
|
60
|
+
(0, 95): 95, (0, 96): 96, (0, 97): 97, (0, 99): 99,
|
|
61
|
+
|
|
62
|
+
# 10 (top) row
|
|
63
|
+
(10, 0): 10, (10, 10): 10, (10, 30): 50, (10, 40): 60,
|
|
64
|
+
(10, 50): 50, (10, 60): 60,
|
|
65
|
+
|
|
66
|
+
# 20 (bottom) row
|
|
67
|
+
(20, 0): 20, (20, 20): 20, (20, 30): 70, (20, 40): 80,
|
|
68
|
+
(20, 70): 70, (20, 80): 80,
|
|
69
|
+
|
|
70
|
+
# 30 (left) row
|
|
71
|
+
(30, 0): 30, (30, 10): 50, (30, 20): 70, (30, 30): 30, (30, 40): 30,
|
|
72
|
+
(30, 50): 50, (30, 60): 50, (30, 70): 70, (30, 80): 70,
|
|
73
|
+
|
|
74
|
+
# 40 (right) row
|
|
75
|
+
(40, 0): 40, (40, 10): 60, (40, 20): 80, (40, 30): 40, (40, 40): 40,
|
|
76
|
+
(40, 50): 60, (40, 60): 60, (40, 70): 80, (40, 80): 80,
|
|
77
|
+
|
|
78
|
+
# 50 (top-left corner) row — invariant under left/top, collapses for right/bottom
|
|
79
|
+
(50, 0): 50, (50, 10): 50, (50, 20): 50, (50, 30): 50, (50, 40): 50,
|
|
80
|
+
(50, 50): 50, (50, 60): 50, (50, 70): 70, (50, 80): 70,
|
|
81
|
+
|
|
82
|
+
# 60 (top-right corner) row
|
|
83
|
+
(60, 0): 60, (60, 10): 60, (60, 20): 60, (60, 30): 60, (60, 40): 60,
|
|
84
|
+
(60, 50): 60, (60, 60): 60, (60, 70): 80, (60, 80): 80,
|
|
85
|
+
|
|
86
|
+
# 70 (bottom-left corner) row
|
|
87
|
+
(70, 0): 70, (70, 10): 50, (70, 20): 70, (70, 30): 70, (70, 40): 70,
|
|
88
|
+
(70, 50): 50, (70, 60): 50, (70, 70): 70, (70, 80): 70,
|
|
89
|
+
|
|
90
|
+
# 80 (bottom-right corner) row
|
|
91
|
+
(80, 0): 80, (80, 10): 60, (80, 20): 80, (80, 30): 80, (80, 40): 80,
|
|
92
|
+
(80, 50): 60, (80, 60): 60, (80, 70): 80, (80, 80): 80,
|
|
93
|
+
|
|
94
|
+
# 9X (surround) row — transparency rule: passes children through
|
|
95
|
+
(90, 0): 0, (91, 0): 0, (92, 0): 0, (93, 0): 0, (94, 0): 0,
|
|
96
|
+
(95, 0): 0, (96, 0): 0, (97, 0): 0, (99, 0): 0,
|
|
97
|
+
(90, 10): 10, (91, 10): 10, (92, 10): 10, (93, 10): 10, (94, 10): 10,
|
|
98
|
+
(95, 10): 10, (96, 10): 10, (97, 10): 10, (99, 10): 10,
|
|
99
|
+
(90, 20): 20, (91, 20): 20, (92, 20): 20, (93, 20): 20, (94, 20): 20,
|
|
100
|
+
(95, 20): 20, (96, 20): 20, (97, 20): 20, (99, 20): 20,
|
|
101
|
+
(90, 30): 30, (91, 30): 30, (92, 30): 30, (93, 30): 30, (94, 30): 30,
|
|
102
|
+
(95, 30): 30, (96, 30): 30, (97, 30): 30, (99, 30): 30,
|
|
103
|
+
(90, 40): 40, (91, 40): 40, (92, 40): 40, (93, 40): 40, (94, 40): 40,
|
|
104
|
+
(95, 40): 40, (96, 40): 40, (97, 40): 40, (99, 40): 40,
|
|
105
|
+
(90, 50): 50, (91, 50): 50, (92, 50): 50, (93, 50): 50, (94, 50): 50,
|
|
106
|
+
(95, 50): 50, (96, 50): 50, (97, 50): 50, (99, 50): 50,
|
|
107
|
+
(90, 60): 60, (91, 60): 60, (92, 60): 60, (93, 60): 60, (94, 60): 60,
|
|
108
|
+
(95, 60): 60, (96, 60): 60, (97, 60): 60, (99, 60): 60,
|
|
109
|
+
(90, 70): 70, (91, 70): 70, (92, 70): 70, (93, 70): 70, (94, 70): 70,
|
|
110
|
+
(95, 70): 70, (96, 70): 70, (97, 70): 70, (99, 70): 70,
|
|
111
|
+
(90, 80): 80, (91, 80): 80, (92, 80): 80, (93, 80): 80, (94, 80): 80,
|
|
112
|
+
(95, 80): 80, (96, 80): 80, (97, 80): 80, (99, 80): 80,
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def compose(parent: int, child: int) -> int:
|
|
117
|
+
"""Compose parent position with child position per spec §2.3 table.
|
|
118
|
+
|
|
119
|
+
Raises:
|
|
120
|
+
CompositionError: if (parent, child) is a geometrically invalid pair.
|
|
121
|
+
"""
|
|
122
|
+
try:
|
|
123
|
+
return COMPOSE_TABLE[(parent, child)]
|
|
124
|
+
except KeyError as e:
|
|
125
|
+
raise CompositionError(
|
|
126
|
+
f"Invalid composition: parent={parent}, child={child}"
|
|
127
|
+
) from e
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cjk-semantic-split
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Recursive atomic decomposition of CJK characters with 9-grid spatial encoding
|
|
5
|
+
Project-URL: Source, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script.git
|
|
6
|
+
Project-URL: Bug Tracker, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/issues
|
|
7
|
+
Author-email: Guanyu Yue <yue.guanyu@hotmail.com>
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: chinese,cjk,decomposition,ideograph
|
|
31
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
32
|
+
Classifier: Operating System :: OS Independent
|
|
33
|
+
Classifier: Programming Language :: Python :: 3
|
|
34
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
39
|
+
Requires-Python: >=3.9
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: hatch>=1.0; extra == 'dev'
|
|
42
|
+
Requires-Dist: hypothesis>=6.0; extra == 'dev'
|
|
43
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
44
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: ruff>=0.1; extra == 'dev'
|
|
46
|
+
Description-Content-Type: text/markdown
|
|
47
|
+
|
|
48
|
+
# cjk-semantic-split
|
|
49
|
+
|
|
50
|
+
Recursive atomic decomposition of CJK characters with 9-grid spatial position encoding.
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install cjk-semantic-split
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from chinese_decompose import decompose
|
|
58
|
+
|
|
59
|
+
decompose("踩")
|
|
60
|
+
# '踩[30]:足[60]:爪[80]:木'
|
|
61
|
+
|
|
62
|
+
decompose("囚")
|
|
63
|
+
# '囚[90]:囗[0]:人'
|
|
64
|
+
|
|
65
|
+
decompose("踩好", as_tree=True)
|
|
66
|
+
# [DecompositionTree('踩'), DecompositionTree('好')]
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Position Encoding
|
|
70
|
+
|
|
71
|
+
Every component sits at a 2-digit position code: cardinals (`10/20/30/40`), single-cell corners (`50/60/70/80`), 4-cell layout strips (`14/16/34/36`), or surround envelopes (`90`–`99`). See the [design spec](docs/superpowers/specs/2026-07-16-cjk-semantic-split-design.md) for full details.
|
|
72
|
+
|
|
73
|
+
## CLI
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
python -m chinese_decompose 踩 # DSL string
|
|
77
|
+
python -m chinese_decompose 踩 --format json # JSON output
|
|
78
|
+
python -m chinese_decompose --self-test # coverage report
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Data Sources
|
|
82
|
+
|
|
83
|
+
Decomposition data is bundled for offline use. Coverage extends across four tiers:
|
|
84
|
+
|
|
85
|
+
- Tier 1 (Unihan kIDS): ~21k chars
|
|
86
|
+
- Tier 2 (wikimedia Commons): ~22k chars
|
|
87
|
+
- Tier 3 (next-layer harvest): ~80k chars
|
|
88
|
+
- Tier 4 (manual patches): dispute-case overrides
|
|
89
|
+
|
|
90
|
+
Last-wins resolution: patches > l3 > l2 > l1.
|
|
91
|
+
|
|
92
|
+
## Configuration
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from chinese_decompose import set_primitives
|
|
96
|
+
|
|
97
|
+
set_primitives({"木", "火", "水", ...}) # process-local override
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT — see `LICENSE`.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
chinese_decompose/__init__.py,sha256=IL1rJLukrojVwZ5r5q1hakFxJDPRJyKwe5Ln0KWntgg,97
|
|
2
|
+
chinese_decompose/__main__.py,sha256=nQ3Bof4LyuuCSEW0dloPbInX1lP_B5GHeJdvket43Tw,3790
|
|
3
|
+
chinese_decompose/api.py,sha256=wN0ZE3qgtjr8R9Xt3Ds6AxpoQ0-KaqFXrNAqq2-BRu4,4538
|
|
4
|
+
chinese_decompose/dsl.py,sha256=30YvTLrKdqDlBsmaGahVCaeZqjZGZaHfnHkLezPCIYc,3844
|
|
5
|
+
chinese_decompose/errors.py,sha256=CB0GrYVJkVqbgwO71npxI7OT5atJ3B5XB3ocHLHbErk,857
|
|
6
|
+
chinese_decompose/loader.py,sha256=Ron4PWFdFTm05pWVFDlJSM_xQBf8AL5JfmV94MzaapI,4083
|
|
7
|
+
chinese_decompose/primitives.py,sha256=K1Mi6UvlLe6Xpdn58NBSPTULhy9O6ciFJZveYzscu4A,1320
|
|
8
|
+
chinese_decompose/split.py,sha256=mbx1d02WesFJ296DHCfS1PRANd4yhqU3HiR-mmUstLk,1858
|
|
9
|
+
chinese_decompose/structures.py,sha256=rvwxuLzFPKWy6otREzgv-fO_5AvfIe0tI2UVf0O3vjY,5137
|
|
10
|
+
chinese_decompose/data/decomp_l1.json,sha256=PkyhKO7K-2YcYWBuMWni1vADV6Wc7Z-y_ZQ51i1cY2U,425
|
|
11
|
+
chinese_decompose/data/decomp_l2.json,sha256=226RW1qW_YelrkKcSAYKYwun-3RZ0rqXlS2TH5A2fkg,51
|
|
12
|
+
chinese_decompose/data/decomp_l3.json,sha256=MxBMGfms0j_wP_WujpyCoa2DisNESnXvfYDJvYqxsoY,129
|
|
13
|
+
chinese_decompose/data/decomp_patches.json,sha256=3PlQwvVUd_fwNm5auYiUdnnmP5xACSGDxgLYuzNWbVs,51
|
|
14
|
+
chinese_decompose/data/primitives.json,sha256=b7ljpLXr3dAQWZLBe-akyWo-fyKDIBNZwOimba5w6yE,1700
|
|
15
|
+
cjk_semantic_split-0.1.0.dist-info/METADATA,sha256=_j5KpPPTjGOp0E_PyONjv9-yggqvplBqgP5ludaYv7o,3889
|
|
16
|
+
cjk_semantic_split-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
17
|
+
cjk_semantic_split-0.1.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
|
|
18
|
+
cjk_semantic_split-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|