cjk-semantic-split 0.1.2__tar.gz → 0.1.3__tar.gz

cjk_semantic_split-0.1.3/.hypothesis/constants/28b72037d99ba6eb

@@ -0,0 +1,4 @@
+# file: /Users/mac/Develop/gitstore/full_brain_intelligence/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/src/chinese_decompose/__init__.py
+# hypothesis_version: 6.156.6
+
+['0.1.3', 'CompositionError', 'DecompositionTree', 'Loader', 'VALID_POSITIONS', '__version__', 'clear_cache', 'compose', 'decompose', 'get_primitives', 'is_primitive', 'make_loader', 'parse_dsl', 'read_structure', 'reset_primitives', 'serialize_tree', 'set_primitives', 'split_one']

cjk_semantic_split-0.1.3/.hypothesis/constants/a80ac1b9c26466c7

@@ -0,0 +1,4 @@
+# file: /Users/mac/Develop/gitstore/full_brain_intelligence/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/src/chinese_decompose/structures.py
+# hypothesis_version: 6.156.6
+
+[]

{cjk_semantic_split-0.1.2 → cjk_semantic_split-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cjk-semantic-split
-Version: 0.1.2
+Version: 0.1.3
 Summary: Recursive atomic decomposition of CJK characters with 9-grid 2-digit spatial encoding. 将汉字递归拆解为原子部件，并在 9 宫格 2 位数字空间编码中标记每个组件的位置。
 Project-URL: Source, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script.git
 Project-URL: Bug Tracker, https://github.com/yueguanyu/Semantic-Character-Splitting-and-Structural-Coding-for-Chinese-Script/issues

{cjk_semantic_split-0.1.2 → cjk_semantic_split-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cjk-semantic-split"
-version = "0.1.2"
+version = "0.1.3"
 description = "Recursive atomic decomposition of CJK characters with 9-grid 2-digit spatial encoding. 将汉字递归拆解为原子部件，并在 9 宫格 2 位数字空间编码中标记每个组件的位置。"
 readme = "README.md"
 requires-python = ">=3.9"

cjk_semantic_split-0.1.3/scripts/harvest.py

@@ -0,0 +1,325 @@
+"""Offline data pipeline (spec §4.3, §9.2).
+
+Fetches CJK decomposition data from a maintained IDS (Ideographic Description
+Sequences) dataset on jsDelivr CDN (mirroring github.com/yi-bai/ids). The
+dataset is MIT-licensed and covers ~102k CJK characters using Unicode IDC
+notation (⿰⿱⿴⿵⿶⿷⿸⿹⿺⿾⿿, plus 3-cell and overlaid variants we skip).
+
+Usage:
+    python -m scripts.harvest --tier l1 \
+        --output ../src/chinese_decompose/data/decomp_l1.json
+
+Source file format (tab-separated):
+
+    <char>\t<IDC-1>\t<IDC-2>\t...
+
+Multiple alternative decompositions (when present) are joined by ";". We take
+the first alternative. Trailing "t" / "K" / "H" / "d" suffixes on each atom
+indicate shape variants — we strip them.
+
+Sub-graphs within a parent IDC (e.g. ⿱⿰日月一) are recursively expanded and
+component positions composed via the parent ⊙ child rule (see
+``docs/component-position-encoding.md``).
+
+IDC → position table (2-cell only supported in v0.x):
+
+    ⿰ A B  → A at 30, B at 40   (left-right)
+    ⿱ A B  → A at 10, B at 20   (top-bottom)
+    ⿴ A B  → A at 90, B at  0   (full surround)
+    ⿵ A B  → A at 93, B at  0   (surround from above)
+    ⿶ A B  → A at 91, B at  0   (surround from below)
+    ⿷ A B  → A at 92, B at  0   (surround from left)
+    ⿸ A B  → A at 95, B at  0   (surround from upper-left)
+    ⿹ A B  → A at 97, B at  0   (surround from upper-right)
+    ⿺ A B  → A at 99, B at  0   (surround from lower-left)
+    ⿾ A B  → A at 94, B at  0   (surround from right)
+    ⿿ A B  → A at 96, B at  0   (surround from lower-right)
+
+3-cell horizontal (⿲), 3-cell vertical (⿳), and overlaid (⿻/⿼/⿽) entries
+are skipped — they require additional position slots outside v0.x schema.
+"""
+from __future__ import annotations
+
+import argparse
+import io
+import json
+import re
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+
+def _import_for_compose():
+    from chinese_decompose.structures import compose as _compose
+    from chinese_decompose.errors import InvalidPositionCodeError
+    return _compose, InvalidPositionCodeError
+
+
+# 2-cell IDC codes → [pos_for_left/top/wall, pos_for_right/bottom/content]
+IDC_TABLE_2_CELL: dict[str, list[int]] = {
+    "⿰": [30, 40],  # ⿰ left-right
+    "⿱": [10, 20],  # ⿱ top-bottom
+    "⿴": [90, 0],   # ⿴ full surround
+    "⿵": [93, 0],   # ⿵ surround above
+    "⿶": [91, 0],   # ⿶ surround below
+    "⿷": [92, 0],   # ⿷ surround left
+    "⿸": [95, 0],   # ⿸ surround upper-left
+    "⿹": [97, 0],   # ⿹ surround upper-right
+    "⿺": [99, 0],   # ⿺ surround lower-left
+    "⿾": [94, 0],   # ⿾ surround right (extension)
+    "⿿": [96, 0],   # ⿿ surround lower-right (extension)
+}
+
+# Leaf characters that are recognized as atoms in the IDS source. CJK Unified
+# Ideographs, Extension A, Extensions B–F (supplementary plane), and the CJK
+# Compatibility Ideographs block. Strokes (single code points in 1E000+) and
+# other auxiliary blocks are excluded — they would dilute the primitives set.
+def _is_data_char(c: str) -> bool:
+    cp = ord(c)
+    return (
+        0x3400 <= cp <= 0x4DBF    # CJK Extension A
+        or 0x4E00 <= cp <= 0x9FFF  # CJK Unified Ideographs
+        or 0x20000 <= cp <= 0x2FFFF  # CJK Extensions B–F + supplementary
+        or 0xF900 <= cp <= 0xFAFF  # CJK Compatibility Ideographs
+    )
+
+
+# Strip trailing shape-variant markers: t, K, H, d, etc.
+# Examples from the source: "一.(T)" → "一"; "一t(t)" → "一"; "乚w一" → "乚, 一" (split)
+# The "w" / "d" / "K" / "t" / "H" markers indicate variant positioning. We strip
+# them and the surrounding parens, leaving just the CJK characters.
+_VARIANT_MARKER = re.compile(r"[\(\)\[\]]")
+_VARIANT_LETTER = re.compile(r"[a-zA-Z]")
+
+
+def _clean_atom_string(s: str) -> str:
+    """Strip parens, letters, dots, and other markers from an IDS atom segment.
+
+    Returns only the CJK / IDC characters. Variants like '一t(t)' → '一'.
+    """
+    out_chars = []
+    for c in s:
+        if _is_data_char(c) or c in IDC_TABLE_2_CELL:
+            out_chars.append(c)
+        # else: drop letters, parens, dots, etc.
+    return "".join(out_chars)
+
+
+def _consume_one(s: str) -> tuple[str | None, str]:
+    """Consume one component from the start of s.
+
+    Returns (component_string, remaining). The component is either a single
+    CJK character or an entire nested IDC subtree.
+    """
+    if not s:
+        return None, ""
+    c = s[0]
+    if c in IDC_TABLE_2_CELL:
+        rest = s[1:]
+        for _ in IDC_TABLE_2_CELL[c]:
+            child, rest = _consume_one(rest)
+            if child is None:
+                return None, ""
+        return c, rest
+    if _is_data_char(c):
+        return c, s[1:]
+    return None, ""
+
+
+def _walk_kids(
+    s: str,
+    parent_position: int,
+    out: list[tuple[str, int]],
+    compose_fn,
+) -> bool:
+    """Recursively expand an IDS string into one-level (atom, final_position) pairs.
+
+    Returns True on success, False if input contains unsupported IDC pattern
+    (caller should skip).
+    """
+    if not s:
+        return True
+    if len(s) == 1:
+        return False
+    idc = s[0]
+    if idc not in IDC_TABLE_2_CELL:
+        # ⿲ ⿳ ⿻ ⿼ ⿽ not supported.
+        return False
+    rest = s[1:]
+    for local_pos in IDC_TABLE_2_CELL[idc]:
+        comp_str, rest = _consume_one(rest)
+        if comp_str is None:
+            return False
+        composed = compose_fn(parent_position, local_pos)
+        if len(comp_str) == 1:
+            out.append((comp_str, composed))
+        else:
+            # Recurse into subtree
+            if not _walk_kids(comp_str, composed, out, compose_fn):
+                return False
+    if rest:
+        return False
+    return True
+
+
+def _parse_kids(s: str, compose_fn) -> list[tuple[str, int]] | None:
+    """Parse IDS value, return one-level decomp list or None if unsupported."""
+    out: list[tuple[str, int]] = []
+    if _walk_kids(s, parent_position=0, out=out, compose_fn=compose_fn):
+        return out
+    return None
+
+
+# --- IDS source download + parse ---
+
+# jsDelivr CDN mirror of github.com/yi-bai/ids (MIT licensed).
+IDS_SOURCES: dict[str, str] = {
+    "lv0": "https://cdn.jsdelivr.net/gh/yi-bai/ids@master/ids_lv0.txt",
+    "lv1": "https://cdn.jsdelivr.net/gh/yi-bai/ids@master/ids_lv1.txt",
+    "lv2": "https://cdn.jsdelivr.net/gh/yi-bai/ids@master/ids_lv2.txt",
+}
+
+
+def fetch_text(url: str) -> str:
+    req = urllib.request.Request(url, headers={"User-Agent": "chinese_decompose/0.1"})
+    with urllib.request.urlopen(req, timeout=120) as resp:
+        return resp.read().decode("utf-8")
+
+
+def parse_ids_source(text: str, compose_fn) -> tuple[dict[str, list[tuple[str, int]]], set[str]]:
+    """Parse yi-bai/ids .txt format, return (entries, primitive_refs).
+
+    Format per line: ``<char>\\t<ids-alt-1>\\t<ids-alt-2>...``
+
+    Each ``ids-alt-N`` is an IDS expression possibly followed by ";alt2;alt3...".
+    We pick the first alt that parses cleanly.
+    """
+    entries: dict[str, list[tuple[str, int]]] = {}
+    refs: set[str] = set()
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        parts = line.split("\t")
+        char_field = parts[0]
+        if len(char_field) != 1 or not _is_data_char(char_field):
+            continue
+        char = char_field
+        # Pick first alt that parses
+        for ids_field in parts[1:]:
+            # Take only the first alt (before ';')
+            first_alt = ids_field.split(";", 1)[0]
+            cleaned = _clean_atom_string(first_alt)
+            if not cleaned:
+                continue
+            result = _parse_kids(cleaned, compose_fn)
+            if result is not None and result:
+                entries[char] = result
+                for c, _ in result:
+                    refs.add(c)
+                break
+    return entries, refs
+
+
+def emit_tier1(output_path: Path, compose_fn) -> dict:
+    """Download IDS dataset, parse, write decomp_l1.json and extend primitives.json."""
+    print(f"Downloading {IDS_SOURCES['lv0']}...", file=sys.stderr)
+    text = fetch_text(IDS_SOURCES["lv0"])
+    print(f"  fetched {len(text.splitlines())} lines", file=sys.stderr)
+
+    print("Parsing IDS entries...", file=sys.stderr)
+    entries, refs = parse_ids_source(text, compose_fn)
+    print(f"  entries: {len(entries)}, primitive refs: {len(refs)}", file=sys.stderr)
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"_meta": {"tier": 1, "source": "github.com/yi-bai/ids", "license": "MIT"}}
+    payload.update({char: [[c, p] for c, p in v] for char, v in entries.items()})
+    output_path.write_text(
+        json.dumps(payload, ensure_ascii=False, separators=(",", ":")), encoding="utf-8"
+    )
+    size_kb = output_path.stat().st_size // 1024
+    print(f"  wrote {output_path} ({size_kb} KB)", file=sys.stderr)
+
+    # Find all characters that get further decomposed in `entries`
+    # (recursively, since chars may appear nested). These MUST be removed
+    # from the primitive set so walker behavior is unambiguous — if a char
+    # is in both primitives and l1, l1 wins (per last-wins loader order).
+    def _collect_decomposable(char: str, seen: set[str], into: set[str]) -> None:
+        if char in seen:
+            return
+        seen.add(char)
+        direct = entries.get(char)
+        if direct is None:
+            return
+        into.add(char)
+        for c, _ in direct:
+            if _is_data_char(c) and len(c) == 1:
+                _collect_decomposable(c, seen, into)
+
+    decomposable: set[str] = set()
+    for char in entries:
+        _collect_decomposable(char, set(), decomposable)
+
+    # Primitive leaves: characters in `refs` that NEVER get decomposed (i.e.,
+    # not in the `decomposable` set we just built). These stay in the
+    # primitive set; the rest get removed.
+    leaves_without_decomp = {
+        c for c in refs if c not in decomposable and _is_data_char(c)
+    }
+
+    # Append new primitives to primitives.json. The bundled primitives.json is
+    # the AUTHORITATIVE atomicity definition (per wikimedia commons' visual
+    # decomposition view): any character in it is treated as atomic, even if
+    # the IDS source happens to have a further decomposition. We only ADD
+    # new leaves — never remove from the curated set.
+    primitives_path = (
+        Path(__file__).parent.parent / "src" / "chinese_decompose" / "data" / "primitives.json"
+    )
+    if primitives_path.exists():
+        existing = set(json.loads(primitives_path.read_text(encoding="utf-8"))["primitives"])
+    else:
+        existing = set()
+    new_primitives = sorted(existing | leaves_without_decomp)
+    primitives_path.write_text(
+        json.dumps(
+            {"primitives": new_primitives},
+            ensure_ascii=False,
+            separators=(",", ":"),
+        ),
+        encoding="utf-8",
+    )
+    added = len(new_primitives) - len(existing)
+    print(
+        f"  appended {added} new primitives (leaves without IDS expansion); "
+        f"total = {len(new_primitives)} "
+        f"({primitives_path.stat().st_size // 1024} KB)",
+        file=sys.stderr,
+    )
+    return entries
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description="Harvest CJK decomposition data.")
+    parser.add_argument("--output", type=Path, required=True, help="Output JSON path.")
+    parser.add_argument(
+        "--tier",
+        choices=["l1"],
+        default="l1",
+        help="Which tier to harvest. Currently only 'l1' (yi-bai/ids lv0) is supported.",
+    )
+    args = parser.parse_args(argv)
+
+    if args.tier != "l1":
+        print(
+            f"Tier {args.tier!r} not implemented; only 'l1' is currently supported.",
+            file=sys.stderr,
+        )
+        return 2
+
+    compose_fn, _ = _import_for_compose()
+    emit_tier1(args.output, compose_fn)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{cjk_semantic_split-0.1.2 → cjk_semantic_split-0.1.3}/src/chinese_decompose/__init__.py

@@ -1,5 +1,5 @@
 """chinese_decompose: recursive atomic decomposition of CJK characters."""
-__version__ = "0.1.2"
+__version__ = "0.1.3"
 
 from chinese_decompose.api import (
     DecompositionTree,