cjk-semantic-split 0.1.4__tar.gz → 0.2.0__tar.gz

{cjk_semantic_split-0.1.4 → cjk_semantic_split-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjk-semantic-split
-Version: 0.1.4
+Version: 0.2.0
 Summary: Recursive atomic decomposition of CJK characters with 9-grid 2-digit spatial encoding. 将汉字递归拆解为原子部件，并在 9 宫格 2 位数字空间编码中标记每个组件的位置。
 Project-URL: Source, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script.git
 Project-URL: Bug Tracker, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/issues

{cjk_semantic_split-0.1.4 → cjk_semantic_split-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cjk-semantic-split"
-version = "0.1.4"
+version = "0.2.0"
 description = "Recursive atomic decomposition of CJK characters with 9-grid 2-digit spatial encoding. 将汉字递归拆解为原子部件，并在 9 宫格 2 位数字空间编码中标记每个组件的位置。"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -40,7 +40,7 @@ dev = [
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/chinese_decompose"]
-include = ["src/chinese_decompose/data/*.json"]
+include = ["src/chinese_decompose/data/*.txt"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]

{cjk_semantic_split-0.1.4 → cjk_semantic_split-0.2.0}/scripts/harvest.py

@@ -7,7 +7,7 @@ notation (⿰⿱⿴⿵⿶⿷⿸⿹⿺⿾⿿, plus 3-cell and overlaid variants w
 
 Usage:
     python -m scripts.harvest --tier l1 \
-        --output ../src/chinese_decompose/data/decomp_l1.json
+        --output ../src/chinese_decompose/data/decomp_l1.txt
 
 Source file format (tab-separated):
 
@@ -153,6 +153,11 @@ def _walk_kids(
             return False
         composed = compose_fn(parent_position, local_pos)
         if len(comp_str) == 1:
+            # Reject leaves that are themselves IDC codes (e.g. degenerate
+            # yi-bai entries where ⿰ appears as a slot — that's a parse
+            # artifact, not a real atom).
+            if comp_str in IDC_TABLE_2_CELL:
+                return False
             out.append((comp_str, composed))
         else:
             # Recurse into subtree
@@ -221,8 +226,40 @@ def parse_ids_source(text: str, compose_fn) -> tuple[dict[str, list[tuple[str, i
     return entries, refs
 
 
+def _serialize_dsl(char: str, comps: list[tuple[str, int]]) -> str:
+    """Build the staggered DSL string for a character + its components.
+
+    Matches the format used by ``DecompositionTree.to_dsl()``:
+    "踩 [30]:足 [60]:爪 [80]:木" — the bracket carries the *next* leaf's
+    position; the last leaf is bare. A primitive char with no children
+    yields just the bare char.
+    """
+    if not comps:
+        return char  # primitive — bare char
+    parts = [f"{char} [{comps[0][1]}]"]
+    leaves = [c for c, _ in comps]
+    leaf_positions = [p for _, p in comps]
+    for i in range(len(leaves) - 1):
+        parts.append(f"{leaves[i]} [{leaf_positions[i + 1]}]")
+    parts.append(leaves[-1])
+    return ":".join(parts)
+
+
+def _load_existing_primitives_txt(path: Path) -> set[str]:
+    if not path.exists():
+        return set()
+    chars: set[str] = set()
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if len(line) == 1:
+            chars.add(line)
+    return chars
+
+
 def emit_tier1(output_path: Path, compose_fn) -> dict:
-    """Download IDS dataset, parse, write decomp_l1.json and extend primitives.json."""
+    """Download IDS dataset, parse, write decomp_l1.txt and extend primitives.txt."""
     print(f"Downloading {IDS_SOURCES['lv0']}...", file=sys.stderr)
     text = fetch_text(IDS_SOURCES["lv0"])
     print(f"  fetched {len(text.splitlines())} lines", file=sys.stderr)
@@ -232,18 +269,26 @@ def emit_tier1(output_path: Path, compose_fn) -> dict:
     print(f"  entries: {len(entries)}, primitive refs: {len(refs)}", file=sys.stderr)
 
     output_path.parent.mkdir(parents=True, exist_ok=True)
-    payload = {"_meta": {"tier": 1, "source": "github.com/yi-bai/ids", "license": "MIT"}}
-    payload.update({char: [[c, p] for c, p in v] for char, v in entries.items()})
-    output_path.write_text(
-        json.dumps(payload, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
-    )
+
+    # Emit Tier 1 as TXT (DSL format, one line per char). Format:
+    #   <char>\t<DSL>
+    # Blank lines and lines starting with `#` are comments.
+    lines = [
+        f"# Tier 1 decomposition data harvested from yi-bai/ids (MIT) via jsDelivr",
+        "# Format per line: <char>\\t<DSL>",
+        "# DSL is the staggered form: each bracket carries the NEXT atom's",
+        "# position; the last atom is bare. Lines starting with # are comments.",
+        "# Empty DSL after the tab means the character is primitive at this tier.",
+    ]
+    for char in sorted(entries):
+        lines.append(f"{char}\t{_serialize_dsl(char, entries[char])}")
+    output_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
     size_kb = output_path.stat().st_size // 1024
     print(f"  wrote {output_path} ({size_kb} KB)", file=sys.stderr)
 
     # Find all characters that get further decomposed in `entries`
-    # (recursively, since chars may appear nested). These MUST be removed
-    # from the primitive set so walker behavior is unambiguous — if a char
-    # is in both primitives and l1, l1 wins (per last-wins loader order).
+    # (recursively, since chars may appear nested). These MUST NOT be
+    # promoted into the primitive set.
     def _collect_decomposable(char: str, seen: set[str], into: set[str]) -> None:
         if char in seen:
             return
@@ -260,33 +305,28 @@ def emit_tier1(output_path: Path, compose_fn) -> dict:
     for char in entries:
         _collect_decomposable(char, set(), decomposable)
 
-    # Primitive leaves: characters in `refs` that NEVER get decomposed (i.e.,
-    # not in the `decomposable` set we just built). These stay in the
-    # primitive set; the rest get removed.
     leaves_without_decomp = {
         c for c in refs if c not in decomposable and _is_data_char(c)
     }
 
-    # Append new primitives to primitives.json. The bundled primitives.json is
-    # the AUTHORITATIVE atomicity definition (per wikimedia commons' visual
-    # decomposition view): any character in it is treated as atomic, even if
-    # the IDS source happens to have a further decomposition. We only ADD
-    # new leaves — never remove from the curated set.
+    # Append new leaves to primitives.txt. Per the wikimedia commons
+    # atomicity principle (§6.3 of the spec), primitives.txt is the
+    # authoritative atomicity definition — we only ADD new leaves; never
+    # remove from the curated set.
     primitives_path = (
-        Path(__file__).parent.parent / "src" / "chinese_decompose" / "data" / "primitives.json"
+        Path(__file__).parent.parent / "src" / "chinese_decompose" / "data" / "primitives.txt"
     )
-    if primitives_path.exists():
-        existing = set(json.loads(primitives_path.read_text(encoding="utf-8"))["primitives"])
-    else:
-        existing = set()
+    existing = _load_existing_primitives_txt(primitives_path)
     new_primitives = sorted(existing | leaves_without_decomp)
+    primitives_lines = [
+        "# chinese-decompose primitives (authoritative atomic components per wikimedia commons view)",
+        "# Format: one character per line. Lines starting with # are comments.",
+        "# Characters listed here are treated as atomic by the walker regardless",
+        "# of any deeper structure recorded in decomp_*.txt.",
+    ]
+    primitives_lines += new_primitives
     primitives_path.write_text(
-        json.dumps(
-            {"primitives": new_primitives},
-            ensure_ascii=False,
-            separators=(",", ":"),
-        ),
-        encoding="utf-8",
+        "\n".join(primitives_lines) + "\n", encoding="utf-8"
     )
     added = len(new_primitives) - len(existing)
     print(

{cjk_semantic_split-0.1.4 → cjk_semantic_split-0.2.0}/src/chinese_decompose/__init__.py

@@ -1,5 +1,5 @@
 """chinese_decompose: recursive atomic decomposition of CJK characters."""
-__version__ = "0.1.4"
+__version__ = "0.2.0"
 
 from chinese_decompose.api import (
     DecompositionTree,