cjk-semantic-split 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ """chinese_decompose: recursive atomic decomposition of CJK characters."""
2
+ __version__ = "0.1.0"
@@ -0,0 +1,134 @@
1
+ """CLI entry point: python -m chinese_decompose."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ from chinese_decompose import __version__
10
+ from chinese_decompose.api import decompose
11
+ from chinese_decompose.errors import ChineseDecomposeError, UnknownCharacterError
12
+ from chinese_decompose.loader import make_loader
13
+
14
+
15
+ def _build_parser() -> argparse.ArgumentParser:
16
+ parser = argparse.ArgumentParser(
17
+ prog="chinese_decompose",
18
+ description="Decompose CJK characters into atomic primitives with spatial positions.",
19
+ )
20
+ parser.add_argument(
21
+ "chars",
22
+ nargs="*",
23
+ help="One or more CJK characters to decompose. If empty, read from stdin.",
24
+ )
25
+ parser.add_argument(
26
+ "--format",
27
+ choices=["dsl", "json", "tree"],
28
+ default="json",
29
+ help="Output format (default: json).",
30
+ )
31
+ parser.add_argument(
32
+ "--no-strict",
33
+ action="store_true",
34
+ help="Emit ? markers for unknown characters instead of raising.",
35
+ )
36
+ parser.add_argument(
37
+ "--primitives",
38
+ type=Path,
39
+ help="Path to a JSON file overriding the primitive set.",
40
+ )
41
+ parser.add_argument(
42
+ "--self-test",
43
+ action="store_true",
44
+ help="Run a self-test reporting tier coverage.",
45
+ )
46
+ parser.add_argument(
47
+ "--version",
48
+ action="version",
49
+ version=f"chinese_decompose {__version__}",
50
+ )
51
+ return parser
52
+
53
+
54
+ def _read_stdin() -> str:
55
+ return sys.stdin.read().strip()
56
+
57
+
58
+ def _format_dsl(text: str, strict: bool) -> str:
59
+ return decompose(text, strict=strict)
60
+
61
+
62
+ def _format_json(text: str, strict: bool) -> str:
63
+ trees = decompose(text, as_tree=True, strict=strict)
64
+ out = []
65
+ for tree in trees:
66
+ out.append({
67
+ "char": tree.char,
68
+ "atoms": list(tree.atoms),
69
+ "positions": list(tree.positions),
70
+ "dsl": tree.to_dsl(),
71
+ })
72
+ return json.dumps(out, ensure_ascii=False, indent=2)
73
+
74
+
75
+ def _format_tree(text: str, strict: bool) -> str:
76
+ trees = decompose(text, as_tree=True, strict=strict)
77
+ return "\n".join(repr(t) for t in trees)
78
+
79
+
80
+ def _self_test() -> str:
81
+ loader = make_loader()
82
+ total = len(loader)
83
+ return (
84
+ f"chinese_decompose self-test\n"
85
+ f" version: {__version__}\n"
86
+ f" characters indexed: {total}\n"
87
+ f" tiers: 4 (l1, l2, l3, patches)\n"
88
+ )
89
+
90
+
91
+ def main(argv: list[str] | None = None) -> int:
92
+ parser = _build_parser()
93
+ args = parser.parse_args(argv)
94
+
95
+ if args.primitives is not None:
96
+ data = json.loads(args.primitives.read_text(encoding="utf-8"))
97
+ from chinese_decompose.primitives import set_primitives
98
+ if isinstance(data, list):
99
+ set_primitives(data)
100
+ elif isinstance(data, dict) and "primitives" in data:
101
+ set_primitives(data["primitives"])
102
+
103
+ if args.self_test:
104
+ sys.stdout.write(_self_test())
105
+ return 0
106
+
107
+ text = "".join(args.chars) if args.chars else _read_stdin()
108
+ if not text:
109
+ parser.print_help()
110
+ return 1
111
+
112
+ strict = not args.no_strict
113
+ try:
114
+ if args.format == "dsl":
115
+ output = _format_dsl(text, strict)
116
+ elif args.format == "json":
117
+ output = _format_json(text, strict)
118
+ elif args.format == "tree":
119
+ output = _format_tree(text, strict)
120
+ else:
121
+ output = ""
122
+ except UnknownCharacterError as e:
123
+ sys.stderr.write(f"Error: UnknownCharacterError: {e}\n")
124
+ return 2
125
+ except ChineseDecomposeError as e:
126
+ sys.stderr.write(f"Error: {type(e).__name__}: {e}\n")
127
+ return 1
128
+
129
+ sys.stdout.write(output + "\n")
130
+ return 0
131
+
132
+
133
+ if __name__ == "__main__":
134
+ sys.exit(main())
@@ -0,0 +1,135 @@
1
+ """Decomposition tree and DSL serialization helpers."""
2
+ from __future__ import annotations
3
+
4
+ from dataclasses import dataclass
5
+ from typing import Iterable, Union
6
+
7
+ from chinese_decompose.errors import UnknownCharacterError
8
+ from chinese_decompose.loader import make_loader as _make_loader
9
+ from chinese_decompose.split import split_one
10
+
11
+
12
+ @dataclass(frozen=True)
13
+ class DecompositionTree:
14
+ """Immutable decomposition result for a single character.
15
+
16
+ Attributes:
17
+ char: The original character (str of length 1).
18
+ children: Tuple of sub-trees at the next level.
19
+ position: The character's own 2-digit position code (set by parent).
20
+
21
+ Derived properties:
22
+ atoms: Flat list of leaf characters (primitives) in DFS order.
23
+ positions: Flat list of leaf positions corresponding to atoms.
24
+ """
25
+
26
+ char: str
27
+ children: tuple["DecompositionTree", ...] = ()
28
+ position: int | None = None
29
+
30
+ def __post_init__(self) -> None:
31
+ if not isinstance(self.char, str) or len(self.char) != 1:
32
+ raise ValueError(
33
+ f"char must be a single-character string, got {self.char!r}"
34
+ )
35
+
36
+ @property
37
+ def atoms(self) -> tuple[str, ...]:
38
+ """Leaf (primitive) characters in DFS decomposition order."""
39
+ if not self.children:
40
+ return (self.char,)
41
+ out: list[str] = []
42
+ for child in self.children:
43
+ out.extend(child.atoms)
44
+ return tuple(out)
45
+
46
+ @property
47
+ def positions(self) -> tuple[int, ...]:
48
+ """Leaf positions corresponding to atoms, in DFS order."""
49
+ if not self.children:
50
+ return (self.position,) if self.position is not None else ()
51
+ out: list[int] = []
52
+ for child in self.children:
53
+ out.extend(child.positions)
54
+ return tuple(out)
55
+
56
+ def to_dsl(self) -> str:
57
+ """Serialize to DSL format per spec §5.1.
58
+
59
+ Staggered format: each segment's bracket carries the NEXT leaf's
60
+ position; the final leaf is bare. A single space is inserted between
61
+ every character (root or leaf) and its following bracketed position,
62
+ e.g. `踩 [30]:足 [60]:爪 [80]:木`. Primitives emit just the char.
63
+ """
64
+ if not self.children:
65
+ return self.char
66
+
67
+ leaves = self.atoms
68
+ leaf_positions = self.positions
69
+
70
+ if not leaf_positions:
71
+ return self.char + ":" + ":".join(leaves)
72
+
73
+ parts: list[str] = [f"{self.char} [{leaf_positions[0]}]"]
74
+ for i in range(len(leaves) - 1):
75
+ parts.append(f"{leaves[i]} [{leaf_positions[i + 1]}]")
76
+ parts.append(leaves[-1])
77
+ return ":".join(parts)
78
+
79
+
80
+ def decompose(
81
+ text: str,
82
+ as_tree: bool = False,
83
+ strict: bool = True,
84
+ loader=None,
85
+ ) -> Union[str, list[DecompositionTree]]:
86
+ """Decompose each character in `text` and return DSL string or list of trees.
87
+
88
+ Args:
89
+ text: String of one or more CJK characters.
90
+ as_tree: If True, return list of DecompositionTree (one per char).
91
+ If False (default), return the flat DSL string.
92
+ strict: If True (default), raise UnknownCharacterError on missing data.
93
+ If False, emit `?[?]:?` markers.
94
+ loader: Optional Loader instance. Defaults to singleton via make_loader().
95
+
96
+ Returns:
97
+ If as_tree=False: a DSL string. Multi-char input joined by `|`.
98
+ If as_tree=True: a list of DecompositionTree (one per char, in input order).
99
+ """
100
+ if loader is None:
101
+ loader = _make_loader()
102
+
103
+ trees: list[DecompositionTree] = []
104
+ for char in text:
105
+ flat = split_one(char, loader)
106
+ if flat is None:
107
+ if strict:
108
+ raise UnknownCharacterError(char)
109
+ trees.append(_marker_tree())
110
+ continue
111
+
112
+ if not flat:
113
+ # Primitive leaf
114
+ trees.append(DecompositionTree(char=char, children=()))
115
+ else:
116
+ children = tuple(
117
+ DecompositionTree(char=atom, children=(), position=pos)
118
+ for atom, pos in flat
119
+ )
120
+ trees.append(DecompositionTree(char=char, children=children))
121
+
122
+ if as_tree:
123
+ return trees
124
+ return "|".join(t.to_dsl() for t in trees)
125
+
126
+
127
+ def _marker_tree() -> DecompositionTree:
128
+ """Marker tree for unknown characters in non-strict mode."""
129
+ return DecompositionTree(
130
+ char="?",
131
+ children=(DecompositionTree(char="?", children=(), position=0),),
132
+ )
133
+
134
+
135
+ __all__ = ["DecompositionTree", "decompose"]
@@ -0,0 +1,12 @@
1
+ {
2
+ "_meta": {"tier": 1, "source": "manual-fixture"},
3
+ "好": [["女", 30], ["子", 40]],
4
+ "森": [["木", 10], ["木", 20], ["木", 40]],
5
+ "囚": [["囗", 90], ["人", 0]],
6
+ "回": [["囗", 90], ["口", 0]],
7
+ "男": [["田", 10], ["力", 20]],
8
+ "休": [["亻", 30], ["木", 40]],
9
+ "明": [["日", 30], ["月", 40]],
10
+ "晶": [["日", 10], ["日", 20], ["日", 40]],
11
+ "品": [["口", 10], ["口", 20], ["口", 40]]
12
+ }
@@ -0,0 +1 @@
1
+ {"_meta": {"tier": 2, "source": "manual-fixture"}}
@@ -0,0 +1,5 @@
1
+ {
2
+ "_meta": {"tier": 3, "source": "manual-fixture"},
3
+ "踩": [["足", 30], ["采", 40]],
4
+ "采": [["爪", 10], ["木", 20]]
5
+ }
@@ -0,0 +1 @@
1
+ {"_meta": {"tier": 4, "source": "manual-fixture"}}
@@ -0,0 +1,21 @@
1
+ {
2
+ "primitives": [
3
+ "木","火","土","金","水","人","口","手","心","日","月","目","耳","鼻","舌","身",
4
+ "爪","戈","矢","石","一","乙","二","三","四","五","六","七","八","九","十","丁",
5
+ "卜","厂","寸","干","弓","牛","犬","玉","王","中","山","川","田","禾","竹","米",
6
+ "糸","网","羊","马","鸟","鱼","虫","贝","车","舟","门","阝","亻","刂","冫","氵",
7
+ "犭","忄","宀","艹","辶","阝","饣","亠","匚","凵","冂","卩","廴","彳","攴","疒",
8
+ "癶","衤","礻","纟","讠","钅","页","风","云","雨","雪","雷","女","子","父","母",
9
+ "兄","弟","姐","妹","己","已","巳","工","土","才","寸","巾","干","幺","广","廴",
10
+ "彐","彡","扌","支","攴","文","斗","斤","方","无","日","曰","月","木","欠","止",
11
+ "歹","殳","毋","比","毛","氏","气","水","火","爪","父","爻","爿","片","牙","牛",
12
+ "犬","玄","玉","瓜","瓦","甘","生","用","田","疋","疒","癶","白","皮","皿","目",
13
+ "矛","矢","石","示","禸","禾","穴","立","竹","米","糸","缶","网","羊","羽","老",
14
+ "而","耒","耳","聿","肉","臣","自","至","臼","舌","舛","舟","艮","色","艸","虍",
15
+ "虫","血","行","衣","襾","覀","角","言","谷","豆","豕","豸","貝","赤","走","足",
16
+ "身","車","辛","辰","辵","邑","酉","釆","里","金","長","門","阜","隶","隹","雨",
17
+ "靑","非","面","革","韋","音","頁","風","飛","食","首","香","馬","骨","高","髟",
18
+ "鬥","鬯","鬲","鬼","魚","鳥","鹵","鹿","麥","麻","黃","黍","黑","黹","黽","鼎",
19
+ "鼓","鼠","鼻","齊","亀","龍","龜","龠","囗"
20
+ ]
21
+ }
@@ -0,0 +1,102 @@
1
+ """DSL serializer and parser per spec §5."""
2
+ from __future__ import annotations
3
+
4
+ import re
5
+ from typing import Iterable, Union
6
+
7
+ from chinese_decompose.api import DecompositionTree
8
+ from chinese_decompose.errors import MalformedDSLParseError
9
+ from chinese_decompose.structures import VALID_POSITIONS
10
+
11
+ # Match a single char (CJK Unified + ASCII fallback) with optional [pos].
12
+ # Allows an optional space between char and [pos] (user-facing format).
13
+ _SEGMENT_PATTERN = re.compile(r"^([一-鿿 -〿?])\s*(?:\[(-?\d+)\])?$")
14
+
15
+
16
+ def serialize_tree(tree: DecompositionTree) -> str:
17
+ """Serialize a single DecompositionTree to DSL.
18
+
19
+ Delegates to DecompositionTree.to_dsl() — same staggered format
20
+ (each segment's bracket is the NEXT leaf's position; last leaf is bare)
21
+ with spaces between chars and brackets.
22
+ """
23
+ return tree.to_dsl()
24
+
25
+
26
+ def serialize_many(trees: Iterable[DecompositionTree]) -> str:
27
+ """Serialize multiple trees joined by `|` (per spec §5.3)."""
28
+ return "|".join(serialize_tree(t) for t in trees)
29
+
30
+
31
+ def _parse_position(raw: str | None) -> int | None:
32
+ if raw is None:
33
+ return None
34
+ try:
35
+ code = int(raw)
36
+ except ValueError as e:
37
+ raise MalformedDSLParseError(f"Position code not an integer: {raw!r}") from e
38
+ if code not in VALID_POSITIONS:
39
+ raise MalformedDSLParseError(f"Position code {code!r} not in valid set")
40
+ return code
41
+
42
+
43
+ def parse_dsl(dsl_str: str) -> DecompositionTree:
44
+ """Parse a DSL string into a DecompositionTree.
45
+
46
+ Format: `<root>[<pos>]:<segment>:<segment>:...:<lastLeaf>`.
47
+
48
+ The last leaf is bare (no brackets). Earlier segments may also lack
49
+ brackets (treated as position=None). Parser is 1-level — produces a flat
50
+ tree where children are leaves (intermediate non-leaf nodes are not
51
+ reconstructed; full round-trip would need richer nesting logic, deferred
52
+ to a future task).
53
+
54
+ Raises:
55
+ MalformedDSLParseError: on empty input, missing/invalid chars,
56
+ non-numeric or out-of-set position codes, or empty segments.
57
+ """
58
+ if not dsl_str:
59
+ raise MalformedDSLParseError("Empty DSL string")
60
+
61
+ raw_segments = dsl_str.split(":")
62
+ if not raw_segments or not raw_segments[0]:
63
+ raise MalformedDSLParseError("Missing root segment")
64
+
65
+ root_match = _SEGMENT_PATTERN.match(raw_segments[0])
66
+ if not root_match:
67
+ raise MalformedDSLParseError(f"Invalid root segment: {raw_segments[0]!r}")
68
+ root_char = root_match.group(1)
69
+ root_position = _parse_position(root_match.group(2))
70
+
71
+ # Detect malformed (missing/extra colons). Bare segments are allowed only
72
+ # for the very last one.
73
+ children: list[DecompositionTree] = []
74
+ for idx, seg in enumerate(raw_segments[1:], start=1):
75
+ last = idx == len(raw_segments) - 1
76
+ if not seg:
77
+ if last and idx == 1:
78
+ # Just `好[30]:` — trailing colon, treat as 1-leaf tree
79
+ break
80
+ raise MalformedDSLParseError(f"Empty segment at index {idx}")
81
+ m = _SEGMENT_PATTERN.match(seg)
82
+ if not m:
83
+ raise MalformedDSLParseError(f"Invalid segment: {seg!r}")
84
+ child_char = m.group(1)
85
+ child_pos_raw = m.group(2)
86
+ if child_pos_raw is None and not last:
87
+ raise MalformedDSLParseError(f"Non-final segment missing position: {seg!r}")
88
+ child_position = _parse_position(child_pos_raw)
89
+ children.append(
90
+ DecompositionTree(char=child_char, children=(), position=child_position)
91
+ )
92
+
93
+ return DecompositionTree(
94
+ char=root_char,
95
+ children=tuple(children),
96
+ position=root_position,
97
+ )
98
+
99
+
100
+ def parse_dsl_multi(dsl_str: str) -> list[DecompositionTree]:
101
+ """Parse a multi-character DSL (joined by `|`)."""
102
+ return [parse_dsl(part) for part in dsl_str.split("|") if part]
@@ -0,0 +1,26 @@
1
+ """Exception hierarchy for chinese_decompose."""
2
+ from __future__ import annotations
3
+
4
+
5
+ class ChineseDecomposeError(Exception):
6
+ """Base class for all chinese_decompose errors."""
7
+
8
+
9
+ class UnknownCharacterError(ChineseDecomposeError):
10
+ """Raised when a character has no entry across all data tiers."""
11
+
12
+ def __init__(self, char: str):
13
+ self.char = char
14
+ super().__init__(f"No decomposition data for character: {char!r}")
15
+
16
+
17
+ class MalformedDSLParseError(ChineseDecomposeError):
18
+ """Raised when DSL parsing fails (bad position, malformed segment)."""
19
+
20
+
21
+ class InvalidPositionCodeError(ChineseDecomposeError):
22
+ """Raised when a data file or input contains a code outside the valid 2-digit set."""
23
+
24
+
25
+ class CompositionError(ChineseDecomposeError):
26
+ """Raised by compose(parent, child) when the combination is geometrically invalid."""
@@ -0,0 +1,132 @@
1
+ """Multi-tier decomposition data loader with last-wins resolution."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ from chinese_decompose.errors import InvalidPositionCodeError
9
+ from chinese_decompose.structures import VALID_POSITIONS
10
+
11
+ _TIER_FILES: dict[str, Path | None] = {}
12
+
13
+
14
+ def _resolve_default_paths() -> tuple[Path, Path, Path, Path]:
15
+ data = Path(__file__).parent / "data"
16
+ return (
17
+ data / "decomp_l1.json",
18
+ data / "decomp_l2.json",
19
+ data / "decomp_l3.json",
20
+ data / "decomp_patches.json",
21
+ )
22
+
23
+
24
+ def _validate_entry(char: str, entries: Any) -> list[tuple[str, int]]:
25
+ if not isinstance(entries, list):
26
+ raise InvalidPositionCodeError(
27
+ f"Entry for {char!r} must be a list of [char, code] pairs"
28
+ )
29
+ result = []
30
+ for entry in entries:
31
+ if not isinstance(entry, list) or len(entry) != 2:
32
+ raise InvalidPositionCodeError(
33
+ f"Entry for {char!r} malformed: {entry!r}"
34
+ )
35
+ sub_char, code = entry
36
+ if not isinstance(sub_char, str) or len(sub_char) != 1:
37
+ raise InvalidPositionCodeError(
38
+ f"Sub-component for {char!r} must be a single char, got {sub_char!r}"
39
+ )
40
+ if not isinstance(code, int) or code not in VALID_POSITIONS:
41
+ raise InvalidPositionCodeError(
42
+ f"Position code for {char!r}: {code!r} not in valid set"
43
+ )
44
+ result.append((sub_char, code))
45
+ return result
46
+
47
+
48
+ def _load_json(path: Path) -> dict[str, list[tuple[str, int]]]:
49
+ if not path.exists():
50
+ return {}
51
+ raw = json.loads(path.read_text(encoding="utf-8"))
52
+ # Strip metadata
53
+ raw.pop("_meta", None)
54
+ out: dict[str, list[tuple[str, int]]] = {}
55
+ for char, entries in raw.items():
56
+ out[char] = _validate_entry(char, entries)
57
+ return out
58
+
59
+
60
+ class Loader:
61
+ """Multi-tier decomposition loader.
62
+
63
+ Resolution order: patches > l3 > l2 > l1 (last-wins).
64
+ Caches results in-process; call clear_cache() to reset.
65
+ """
66
+
67
+ _instance: "Loader | None" = None
68
+
69
+ def __init__(
70
+ self,
71
+ l1_path: Path,
72
+ l2_path: Path,
73
+ l3_path: Path,
74
+ patches_path: Path,
75
+ ):
76
+ self._paths = (l1_path, l2_path, l3_path, patches_path)
77
+ self._cache: dict[str, list[tuple[str, int]] | None] = {}
78
+ # Pre-load all data; raise on invalid codes
79
+ l1 = _load_json(l1_path)
80
+ l2 = _load_json(l2_path)
81
+ l3 = _load_json(l3_path)
82
+ patches = _load_json(patches_path)
83
+ self._merged = {}
84
+ for char in l1:
85
+ self._merged[char] = l1[char]
86
+ for char in l2:
87
+ self._merged[char] = l2[char]
88
+ for char in l3:
89
+ self._merged[char] = l3[char]
90
+ for char in patches:
91
+ self._merged[char] = patches[char]
92
+
93
+ def get(self, char: str) -> list[tuple[str, int]] | None:
94
+ """Return one-level decomposition for `char`, or None if unknown."""
95
+ if char in self._cache:
96
+ return self._cache[char]
97
+ result = self._merged.get(char)
98
+ self._cache[char] = result
99
+ return result
100
+
101
+ def __len__(self) -> int:
102
+ return len(self._merged)
103
+
104
+
105
+ def make_loader(
106
+ l1_path: Path | None = None,
107
+ l2_path: Path | None = None,
108
+ l3_path: Path | None = None,
109
+ patches_path: Path | None = None,
110
+ ) -> Loader:
111
+ """Construct (or return cached singleton) Loader instance."""
112
+ if (
113
+ l1_path is None
114
+ and l2_path is None
115
+ and l3_path is None
116
+ and patches_path is None
117
+ ):
118
+ p1, p2, p3, pp = _resolve_default_paths()
119
+ else:
120
+ d1, d2, d3, dp = _resolve_default_paths()
121
+ p1 = l1_path or d1
122
+ p2 = l2_path or d2
123
+ p3 = l3_path or d3
124
+ pp = patches_path or dp
125
+ Loader._instance = Loader(p1, p2, p3, pp)
126
+ return Loader._instance
127
+
128
+
129
+ def clear_cache() -> None:
130
+ """Reset loader cache; next get() rebuilds from JSON."""
131
+ if Loader._instance is not None:
132
+ Loader._instance._cache.clear()
@@ -0,0 +1,41 @@
1
+ """Primitive set: lookup and process-local override."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from pathlib import Path
6
+ from typing import Iterable
7
+
8
+ _DATA_PATH = Path(__file__).parent / "data" / "primitives.json"
9
+ _DEFAULT: frozenset[str] = frozenset(
10
+ json.loads(_DATA_PATH.read_text(encoding="utf-8"))["primitives"]
11
+ )
12
+ _OVERRIDE: frozenset[str] | None = None
13
+
14
+
15
+ def get_primitives() -> frozenset[str]:
16
+ """Return the currently-active primitive set (override or default)."""
17
+ return _OVERRIDE if _OVERRIDE is not None else _DEFAULT
18
+
19
+
20
+ def set_primitives(iterable: Iterable[str]) -> None:
21
+ """Replace the primitive set for the current process.
22
+
23
+ Pass-through: useful for testing with custom primitive subsets.
24
+ Reset by passing None or by re-importing the module. Does NOT persist.
25
+ """
26
+ global _OVERRIDE
27
+ items = list(iterable)
28
+ if not all(isinstance(s, str) and len(s) == 1 for s in items):
29
+ raise ValueError("primitives must be single-character strings")
30
+ _OVERRIDE = frozenset(items)
31
+
32
+
33
+ def is_primitive(char: str) -> bool:
34
+ """Return True if `char` is in the active primitive set."""
35
+ return char in get_primitives()
36
+
37
+
38
+ def reset_primitives() -> None:
39
+ """Clear any process-local override, restoring bundled defaults."""
40
+ global _OVERRIDE
41
+ _OVERRIDE = None
@@ -0,0 +1,52 @@
1
+ """Recursive decomposition walker with position composition."""
2
+ from __future__ import annotations
3
+
4
+ from typing import TYPE_CHECKING
5
+
6
+ from chinese_decompose.primitives import is_primitive
7
+ from chinese_decompose.structures import compose
8
+
9
+ if TYPE_CHECKING:
10
+ from chinese_decompose.loader import Loader
11
+
12
+
13
+ def split_one(char: str, loader: "Loader") -> list[tuple[str, int]] | None:
14
+ """Recursively decompose `char` into (atom_char, final_position) pairs.
15
+
16
+ Returns:
17
+ - [] if `char` is a primitive (no further decomposition).
18
+ - list of (atom_char, position) pairs after full recursive expansion.
19
+ - None if `char` has no entry in the loader.
20
+
21
+ The recursion chains position composition: for each sub-character at its
22
+ declared local position, compose with the parent's position to derive the
23
+ final 9-grid position.
24
+ """
25
+ return _split_recursive(char, parent_position=0, loader=loader)
26
+
27
+
28
+ def _split_recursive(
29
+ char: str, parent_position: int, loader: "Loader"
30
+ ) -> list[tuple[str, int]] | None:
31
+ """Recursively walk the decomposition tree, composing positions."""
32
+ if is_primitive(char):
33
+ # Primitives terminate; no expansion. Empty list signals leaf.
34
+ return []
35
+
36
+ decomp = loader.get(char)
37
+ if decomp is None:
38
+ return None # unknown
39
+
40
+ out: list[tuple[str, int]] = []
41
+ for sub_char, local_pos in decomp:
42
+ final_pos = compose(parent_position, local_pos)
43
+ sub_result = _split_recursive(sub_char, parent_position=final_pos, loader=loader)
44
+ if sub_result is None:
45
+ # Sub-char unknown at this level; bubble up
46
+ return None
47
+ if sub_result:
48
+ out.extend(sub_result)
49
+ else:
50
+ # Primitive sub-character: emit just the char with final position
51
+ out.append((sub_char, final_pos))
52
+ return out
@@ -0,0 +1,127 @@
1
+ """Position code constants for the 9-grid spatial encoding.
2
+
3
+ Codes are 2-digit numbers (spec §2.1): digit A is the spatial zone, digit B
4
+ is the sub-zone within the parent zone (0 means atomic slot, non-zero for
5
+ specific sub-strips or surround variants).
6
+ """
7
+ from __future__ import annotations
8
+
9
+ from chinese_decompose.errors import CompositionError
10
+
11
+ # Atomic single-slot positions
12
+ P_CENTER = 0
13
+ P_TOP = 10
14
+ P_BOTTOM = 20
15
+ P_LEFT = 30
16
+ P_RIGHT = 40
17
+ P_TOP_LEFT = 50
18
+ P_TOP_RIGHT = 60
19
+ P_BOTTOM_LEFT = 70
20
+ P_BOTTOM_RIGHT = 80
21
+
22
+ # 4-cell layout strip positions
23
+ P_STRIP_VERT_RIGHT = 14 # inner-right-edge of upper zone (e.g. 目's right strip)
24
+ P_STRIP_VERT_LEFT = 16 # inner-left-edge of lower zone
25
+ P_STRIP_HORIZ_RIGHT = 34 # inner-right-edge of left zone (e.g. 罒's right strip)
26
+ P_STRIP_HORIZ_LEFT = 36 # inner-left-edge of right zone
27
+
28
+ # Surround envelopes (90-99)
29
+ P_SURROUND_FULL = 90
30
+ P_SURROUND_BELOW = 91
31
+ P_SURROUND_LEFT = 92
32
+ P_SURROUND_ABOVE = 93
33
+ P_SURROUND_RIGHT = 94
34
+ P_SURROUND_UL = 95
35
+ P_SURROUND_LR = 96
36
+ P_SURROUND_UR = 97
37
+ P_SURROUND_LL_LO = 98 # lower-bottom extension (semantics TBD per spec)
38
+ P_SURROUND_LL = 99
39
+
40
+ VALID_POSITIONS: frozenset[int] = frozenset({
41
+ P_CENTER, P_TOP, P_BOTTOM, P_LEFT, P_RIGHT,
42
+ P_TOP_LEFT, P_TOP_RIGHT, P_BOTTOM_LEFT, P_BOTTOM_RIGHT,
43
+ P_STRIP_VERT_RIGHT, P_STRIP_VERT_LEFT,
44
+ P_STRIP_HORIZ_RIGHT, P_STRIP_HORIZ_LEFT,
45
+ P_SURROUND_FULL, P_SURROUND_BELOW, P_SURROUND_LEFT,
46
+ P_SURROUND_ABOVE, P_SURROUND_RIGHT, P_SURROUND_UL,
47
+ P_SURROUND_LR, P_SURROUND_UR, P_SURROUND_LL,
48
+ # NOTE: P_SURROUND_LL_LO (98) intentionally excluded — semantics TBD per spec §2.1
49
+ })
50
+
51
+
52
+ # Composition table (spec §2.3): (parent, child) -> composed position.
53
+ # Missing keys are geometrically invalid and raise CompositionError at runtime.
54
+ COMPOSE_TABLE: dict[tuple[int, int], int] = {
55
+ # 00 (center) row — center passes any valid position through unchanged.
56
+ (0, 0): 0, (0, 10): 10, (0, 20): 20, (0, 30): 30, (0, 40): 40,
57
+ (0, 50): 50, (0, 60): 60, (0, 70): 70, (0, 80): 80,
58
+ (0, 14): 14, (0, 16): 16, (0, 34): 34, (0, 36): 36,
59
+ (0, 90): 90, (0, 91): 91, (0, 92): 92, (0, 93): 93, (0, 94): 94,
60
+ (0, 95): 95, (0, 96): 96, (0, 97): 97, (0, 99): 99,
61
+
62
+ # 10 (top) row
63
+ (10, 0): 10, (10, 10): 10, (10, 30): 50, (10, 40): 60,
64
+ (10, 50): 50, (10, 60): 60,
65
+
66
+ # 20 (bottom) row
67
+ (20, 0): 20, (20, 20): 20, (20, 30): 70, (20, 40): 80,
68
+ (20, 70): 70, (20, 80): 80,
69
+
70
+ # 30 (left) row
71
+ (30, 0): 30, (30, 10): 50, (30, 20): 70, (30, 30): 30, (30, 40): 30,
72
+ (30, 50): 50, (30, 60): 50, (30, 70): 70, (30, 80): 70,
73
+
74
+ # 40 (right) row
75
+ (40, 0): 40, (40, 10): 60, (40, 20): 80, (40, 30): 40, (40, 40): 40,
76
+ (40, 50): 60, (40, 60): 60, (40, 70): 80, (40, 80): 80,
77
+
78
+ # 50 (top-left corner) row — invariant under left/top, collapses for right/bottom
79
+ (50, 0): 50, (50, 10): 50, (50, 20): 50, (50, 30): 50, (50, 40): 50,
80
+ (50, 50): 50, (50, 60): 50, (50, 70): 70, (50, 80): 70,
81
+
82
+ # 60 (top-right corner) row
83
+ (60, 0): 60, (60, 10): 60, (60, 20): 60, (60, 30): 60, (60, 40): 60,
84
+ (60, 50): 60, (60, 60): 60, (60, 70): 80, (60, 80): 80,
85
+
86
+ # 70 (bottom-left corner) row
87
+ (70, 0): 70, (70, 10): 50, (70, 20): 70, (70, 30): 70, (70, 40): 70,
88
+ (70, 50): 50, (70, 60): 50, (70, 70): 70, (70, 80): 70,
89
+
90
+ # 80 (bottom-right corner) row
91
+ (80, 0): 80, (80, 10): 60, (80, 20): 80, (80, 30): 80, (80, 40): 80,
92
+ (80, 50): 60, (80, 60): 60, (80, 70): 80, (80, 80): 80,
93
+
94
+ # 9X (surround) row — transparency rule: passes children through
95
+ (90, 0): 0, (91, 0): 0, (92, 0): 0, (93, 0): 0, (94, 0): 0,
96
+ (95, 0): 0, (96, 0): 0, (97, 0): 0, (99, 0): 0,
97
+ (90, 10): 10, (91, 10): 10, (92, 10): 10, (93, 10): 10, (94, 10): 10,
98
+ (95, 10): 10, (96, 10): 10, (97, 10): 10, (99, 10): 10,
99
+ (90, 20): 20, (91, 20): 20, (92, 20): 20, (93, 20): 20, (94, 20): 20,
100
+ (95, 20): 20, (96, 20): 20, (97, 20): 20, (99, 20): 20,
101
+ (90, 30): 30, (91, 30): 30, (92, 30): 30, (93, 30): 30, (94, 30): 30,
102
+ (95, 30): 30, (96, 30): 30, (97, 30): 30, (99, 30): 30,
103
+ (90, 40): 40, (91, 40): 40, (92, 40): 40, (93, 40): 40, (94, 40): 40,
104
+ (95, 40): 40, (96, 40): 40, (97, 40): 40, (99, 40): 40,
105
+ (90, 50): 50, (91, 50): 50, (92, 50): 50, (93, 50): 50, (94, 50): 50,
106
+ (95, 50): 50, (96, 50): 50, (97, 50): 50, (99, 50): 50,
107
+ (90, 60): 60, (91, 60): 60, (92, 60): 60, (93, 60): 60, (94, 60): 60,
108
+ (95, 60): 60, (96, 60): 60, (97, 60): 60, (99, 60): 60,
109
+ (90, 70): 70, (91, 70): 70, (92, 70): 70, (93, 70): 70, (94, 70): 70,
110
+ (95, 70): 70, (96, 70): 70, (97, 70): 70, (99, 70): 70,
111
+ (90, 80): 80, (91, 80): 80, (92, 80): 80, (93, 80): 80, (94, 80): 80,
112
+ (95, 80): 80, (96, 80): 80, (97, 80): 80, (99, 80): 80,
113
+ }
114
+
115
+
116
+ def compose(parent: int, child: int) -> int:
117
+ """Compose parent position with child position per spec §2.3 table.
118
+
119
+ Raises:
120
+ CompositionError: if (parent, child) is a geometrically invalid pair.
121
+ """
122
+ try:
123
+ return COMPOSE_TABLE[(parent, child)]
124
+ except KeyError as e:
125
+ raise CompositionError(
126
+ f"Invalid composition: parent={parent}, child={child}"
127
+ ) from e
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.4
2
+ Name: cjk-semantic-split
3
+ Version: 0.1.0
4
+ Summary: Recursive atomic decomposition of CJK characters with 9-grid spatial encoding
5
+ Project-URL: Source, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script.git
6
+ Project-URL: Bug Tracker, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/issues
7
+ Author-email: Guanyu Yue <yue.guanyu@hotmail.com>
8
+ License: MIT License
9
+
10
+ Copyright (c) 2026
11
+
12
+ Permission is hereby granted, free of charge, to any person obtaining a copy
13
+ of this software and associated documentation files (the "Software"), to deal
14
+ in the Software without restriction, including without limitation the rights
15
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
16
+ copies of the Software, and to permit persons to whom the Software is
17
+ furnished to do so, subject to the following conditions:
18
+
19
+ The above copyright notice and this permission notice shall be included in all
20
+ copies or substantial portions of the Software.
21
+
22
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
23
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
25
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
27
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
28
+ SOFTWARE.
29
+ License-File: LICENSE
30
+ Keywords: chinese,cjk,decomposition,ideograph
31
+ Classifier: License :: OSI Approved :: MIT License
32
+ Classifier: Operating System :: OS Independent
33
+ Classifier: Programming Language :: Python :: 3
34
+ Classifier: Programming Language :: Python :: 3.9
35
+ Classifier: Programming Language :: Python :: 3.10
36
+ Classifier: Programming Language :: Python :: 3.11
37
+ Classifier: Programming Language :: Python :: 3.12
38
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
39
+ Requires-Python: >=3.9
40
+ Provides-Extra: dev
41
+ Requires-Dist: hatch>=1.0; extra == 'dev'
42
+ Requires-Dist: hypothesis>=6.0; extra == 'dev'
43
+ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
44
+ Requires-Dist: pytest>=7.0; extra == 'dev'
45
+ Requires-Dist: ruff>=0.1; extra == 'dev'
46
+ Description-Content-Type: text/markdown
47
+
48
+ # cjk-semantic-split
49
+
50
+ Recursive atomic decomposition of CJK characters with 9-grid spatial position encoding.
51
+
52
+ ```bash
53
+ pip install cjk-semantic-split
54
+ ```
55
+
56
+ ```python
57
+ from chinese_decompose import decompose
58
+
59
+ decompose("踩")
60
+ # '踩[30]:足[60]:爪[80]:木'
61
+
62
+ decompose("囚")
63
+ # '囚[90]:囗[0]:人'
64
+
65
+ decompose("踩好", as_tree=True)
66
+ # [DecompositionTree('踩'), DecompositionTree('好')]
67
+ ```
68
+
69
+ ## Position Encoding
70
+
71
+ Every component sits at a 2-digit position code: cardinals (`10/20/30/40`), single-cell corners (`50/60/70/80`), 4-cell layout strips (`14/16/34/36`), or surround envelopes (`90`–`99`). See the [design spec](docs/superpowers/specs/2026-07-16-cjk-semantic-split-design.md) for full details.
72
+
73
+ ## CLI
74
+
75
+ ```bash
76
+ python -m chinese_decompose 踩 # DSL string
77
+ python -m chinese_decompose 踩 --format json # JSON output
78
+ python -m chinese_decompose --self-test # coverage report
79
+ ```
80
+
81
+ ## Data Sources
82
+
83
+ Decomposition data is bundled for offline use. Coverage extends across four tiers:
84
+
85
+ - Tier 1 (Unihan kIDS): ~21k chars
86
+ - Tier 2 (wikimedia Commons): ~22k chars
87
+ - Tier 3 (next-layer harvest): ~80k chars
88
+ - Tier 4 (manual patches): dispute-case overrides
89
+
90
+ Last-wins resolution: patches > l3 > l2 > l1.
91
+
92
+ ## Configuration
93
+
94
+ ```python
95
+ from chinese_decompose import set_primitives
96
+
97
+ set_primitives({"木", "火", "水", ...}) # process-local override
98
+ ```
99
+
100
+ ## License
101
+
102
+ MIT — see `LICENSE`.
@@ -0,0 +1,18 @@
1
+ chinese_decompose/__init__.py,sha256=IL1rJLukrojVwZ5r5q1hakFxJDPRJyKwe5Ln0KWntgg,97
2
+ chinese_decompose/__main__.py,sha256=nQ3Bof4LyuuCSEW0dloPbInX1lP_B5GHeJdvket43Tw,3790
3
+ chinese_decompose/api.py,sha256=wN0ZE3qgtjr8R9Xt3Ds6AxpoQ0-KaqFXrNAqq2-BRu4,4538
4
+ chinese_decompose/dsl.py,sha256=30YvTLrKdqDlBsmaGahVCaeZqjZGZaHfnHkLezPCIYc,3844
5
+ chinese_decompose/errors.py,sha256=CB0GrYVJkVqbgwO71npxI7OT5atJ3B5XB3ocHLHbErk,857
6
+ chinese_decompose/loader.py,sha256=Ron4PWFdFTm05pWVFDlJSM_xQBf8AL5JfmV94MzaapI,4083
7
+ chinese_decompose/primitives.py,sha256=K1Mi6UvlLe6Xpdn58NBSPTULhy9O6ciFJZveYzscu4A,1320
8
+ chinese_decompose/split.py,sha256=mbx1d02WesFJ296DHCfS1PRANd4yhqU3HiR-mmUstLk,1858
9
+ chinese_decompose/structures.py,sha256=rvwxuLzFPKWy6otREzgv-fO_5AvfIe0tI2UVf0O3vjY,5137
10
+ chinese_decompose/data/decomp_l1.json,sha256=PkyhKO7K-2YcYWBuMWni1vADV6Wc7Z-y_ZQ51i1cY2U,425
11
+ chinese_decompose/data/decomp_l2.json,sha256=226RW1qW_YelrkKcSAYKYwun-3RZ0rqXlS2TH5A2fkg,51
12
+ chinese_decompose/data/decomp_l3.json,sha256=MxBMGfms0j_wP_WujpyCoa2DisNESnXvfYDJvYqxsoY,129
13
+ chinese_decompose/data/decomp_patches.json,sha256=3PlQwvVUd_fwNm5auYiUdnnmP5xACSGDxgLYuzNWbVs,51
14
+ chinese_decompose/data/primitives.json,sha256=b7ljpLXr3dAQWZLBe-akyWo-fyKDIBNZwOimba5w6yE,1700
15
+ cjk_semantic_split-0.1.0.dist-info/METADATA,sha256=_j5KpPPTjGOp0E_PyONjv9-yggqvplBqgP5ludaYv7o,3889
16
+ cjk_semantic_split-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
17
+ cjk_semantic_split-0.1.0.dist-info/licenses/LICENSE,sha256=ESYyLizI0WWtxMeS7rGVcX3ivMezm-HOd5WdeOh-9oU,1056
18
+ cjk_semantic_split-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.31.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.