cjkfold 0.1.0__py3-none-any.whl

cjkfold/__init__.py

@@ -0,0 +1,36 @@
+"""cjkfold -- fold look-alike CJK glyphs so extracted text matches again.
+
+    >>> from cjkfold import fold
+    >>> fold("⻆⺁⼀") == "角厂一"
+    True
+
+See :mod:`cjkfold.normalize` for the background, :mod:`cjkfold.score` for the
+retrieval-ranking helper that pairs with it, and :mod:`cjkfold.integrations` for
+framework-agnostic adapters that fold a whole document set.
+"""
+
+from __future__ import annotations
+
+from ._table import EQUIVALENT_IDEOGRAPH
+from .integrations import fold_document, fold_documents, rerank, text_of
+from .normalize import Lookalike, fold, has_lookalikes, iter_lookalikes, kind_of
+from .score import exact_boost, ngram_overlap, ngrams
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "fold",
+    "has_lookalikes",
+    "iter_lookalikes",
+    "Lookalike",
+    "kind_of",
+    "ngrams",
+    "ngram_overlap",
+    "exact_boost",
+    "text_of",
+    "fold_document",
+    "fold_documents",
+    "rerank",
+    "EQUIVALENT_IDEOGRAPH",
+    "__version__",
+]

cjkfold/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

cjkfold/_table.py

@@ -0,0 +1,361 @@
+"""Glyph equivalence table. GENERATED — do not edit by hand.
+
+Regenerate:  python scripts/build_table.py
+Source:      Unicode Character Database, EquivalentUnifiedIdeograph.txt
+             https://www.unicode.org/Public/UCD/latest/ucd/EquivalentUnifiedIdeograph.txt
+"""
+
+from __future__ import annotations
+
+# Characters that are visually identical to a unified ideograph but carry a
+# different code point.  NFKC does NOT fold most of these (see README).
+EQUIVALENT_IDEOGRAPH: dict[str, str] = {
+    "\u2e81": "\u5382",  # ⺁ CJK RADICAL CLIFF -> 厂
+    "\u2e82": "\u4e5b",  # ⺂ CJK RADICAL SECOND ONE -> 乛
+    "\u2e83": "\u4e5a",  # ⺃ CJK RADICAL SECOND TWO -> 乚
+    "\u2e84": "\u4e59",  # ⺄ CJK RADICAL SECOND THREE -> 乙
+    "\u2e85": "\u4ebb",  # ⺅ CJK RADICAL PERSON -> 亻
+    "\u2e86": "\u5182",  # ⺆ CJK RADICAL BOX -> 冂
+    "\u2e87": "\U00020628",  # ⺇ CJK RADICAL TABLE -> 𠘨  [U+20628]
+    "\u2e88": "\u5200",  # ⺈ CJK RADICAL KNIFE ONE -> 刀
+    "\u2e89": "\u5202",  # ⺉ CJK RADICAL KNIFE TWO -> 刂
+    "\u2e8a": "\u535c",  # ⺊ CJK RADICAL DIVINATION -> 卜
+    "\u2e8b": "\u353e",  # ⺋ CJK RADICAL SEAL -> 㔾
+    "\u2e8c": "\u5c0f",  # ⺌ CJK RADICAL SMALL ONE -> 小
+    "\u2e8d": "\u5c0f",  # ⺍ CJK RADICAL SMALL TWO -> 小
+    "\u2e8e": "\u5140",  # ⺎ CJK RADICAL LAME ONE -> 兀
+    "\u2e8f": "\u5c23",  # ⺏ CJK RADICAL LAME TWO -> 尣
+    "\u2e90": "\u5c22",  # ⺐ CJK RADICAL LAME THREE -> 尢
+    "\u2e91": "\U00021bc2",  # ⺑ CJK RADICAL LAME FOUR -> 𡯂  [U+21BC2]
+    "\u2e92": "\u5df3",  # ⺒ CJK RADICAL SNAKE -> 巳
+    "\u2e93": "\u5e7a",  # ⺓ CJK RADICAL THREAD -> 幺
+    "\u2e94": "\u5f51",  # ⺔ CJK RADICAL SNOUT ONE -> 彑
+    "\u2e95": "\U0002b739",  # ⺕ CJK RADICAL SNOUT TWO -> 𫜹  [U+2B739]
+    "\u2e96": "\u5fc4",  # ⺖ CJK RADICAL HEART ONE -> 忄
+    "\u2e97": "\u5fc3",  # ⺗ CJK RADICAL HEART TWO -> 心
+    "\u2e98": "\u624c",  # ⺘ CJK RADICAL HAND -> 扌
+    "\u2e99": "\u6535",  # ⺙ CJK RADICAL RAP -> 攵
+    "\u2e9b": "\u65e1",  # ⺛ CJK RADICAL CHOKE -> 旡
+    "\u2e9c": "\u65e5",  # ⺜ CJK RADICAL SUN -> 日
+    "\u2e9d": "\u6708",  # ⺝ CJK RADICAL MOON -> 月
+    "\u2e9e": "\u6b7a",  # ⺞ CJK RADICAL DEATH -> 歺
+    "\u2e9f": "\u6bcd",  # ⺟ CJK RADICAL MOTHER -> 母
+    "\u2ea0": "\u6c11",  # ⺠ CJK RADICAL CIVILIAN -> 民
+    "\u2ea1": "\u6c35",  # ⺡ CJK RADICAL WATER ONE -> 氵
+    "\u2ea2": "\u6c3a",  # ⺢ CJK RADICAL WATER TWO -> 氺
+    "\u2ea3": "\u706c",  # ⺣ CJK RADICAL FIRE -> 灬
+    "\u2ea4": "\u722b",  # ⺤ CJK RADICAL PAW ONE -> 爫
+    "\u2ea5": "\u722b",  # ⺥ CJK RADICAL PAW TWO -> 爫
+    "\u2ea6": "\u4e2c",  # ⺦ CJK RADICAL SIMPLIFIED HALF TREE TRUNK -> 丬
+    "\u2ea7": "\u725b",  # ⺧ CJK RADICAL COW -> 牛
+    "\u2ea8": "\u72ad",  # ⺨ CJK RADICAL DOG -> 犭
+    "\u2ea9": "\u738b",  # ⺩ CJK RADICAL JADE -> 王
+    "\u2eaa": "\U00024d14",  # ⺪ CJK RADICAL BOLT OF CLOTH -> 𤴔  [U+24D14]
+    "\u2eab": "\u76ee",  # ⺫ CJK RADICAL EYE -> 目
+    "\u2eac": "\u793a",  # ⺬ CJK RADICAL SPIRIT ONE -> 示
+    "\u2ead": "\u793b",  # ⺭ CJK RADICAL SPIRIT TWO -> 礻
+    "\u2eae": "\U00025ad7",  # ⺮ CJK RADICAL BAMBOO -> 𥫗  [U+25AD7]
+    "\u2eaf": "\u7cf9",  # ⺯ CJK RADICAL SILK -> 糹
+    "\u2eb0": "\u7e9f",  # ⺰ CJK RADICAL C-SIMPLIFIED SILK -> 纟
+    "\u2eb1": "\u7f53",  # ⺱ CJK RADICAL NET ONE -> 罓
+    "\u2eb2": "\u7f52",  # ⺲ CJK RADICAL NET TWO -> 罒
+    "\u2eb3": "\u34c1",  # ⺳ CJK RADICAL NET THREE -> 㓁
+    "\u2eb4": "\u5197",  # ⺴ CJK RADICAL NET FOUR -> 冗
+    "\u2eb5": "\U0002626b",  # ⺵ CJK RADICAL MESH -> 𦉫  [U+2626B]
+    "\u2eb6": "\u7f8a",  # ⺶ CJK RADICAL SHEEP -> 羊
+    "\u2eb7": "\U0002634c",  # ⺷ CJK RADICAL RAM -> 𦍌  [U+2634C]
+    "\u2eb8": "\U0002634b",  # ⺸ CJK RADICAL EWE -> 𦍋  [U+2634B]
+    "\u2eb9": "\u8002",  # ⺹ CJK RADICAL OLD -> 耂
+    "\u2eba": "\u8080",  # ⺺ CJK RADICAL BRUSH ONE -> 肀
+    "\u2ebb": "\u807f",  # ⺻ CJK RADICAL BRUSH TWO -> 聿
+    "\u2ebc": "\u8089",  # ⺼ CJK RADICAL MEAT -> 肉
+    "\u2ebd": "\U00026951",  # ⺽ CJK RADICAL MORTAR -> 𦥑  [U+26951]
+    "\u2ebe": "\u8279",  # ⺾ CJK RADICAL GRASS ONE -> 艹
+    "\u2ebf": "\u8279",  # ⺿ CJK RADICAL GRASS TWO -> 艹
+    "\u2ec0": "\u8279",  # ⻀ CJK RADICAL GRASS THREE -> 艹
+    "\u2ec1": "\u864e",  # ⻁ CJK RADICAL TIGER -> 虎
+    "\u2ec2": "\u8864",  # ⻂ CJK RADICAL CLOTHES -> 衤
+    "\u2ec3": "\u8980",  # ⻃ CJK RADICAL WEST ONE -> 覀
+    "\u2ec4": "\u897f",  # ⻄ CJK RADICAL WEST TWO -> 西
+    "\u2ec5": "\u89c1",  # ⻅ CJK RADICAL C-SIMPLIFIED SEE -> 见
+    "\u2ec6": "\u89d2",  # ⻆ CJK RADICAL SIMPLIFIED HORN -> 角
+    "\u2ec7": "\U000278b2",  # ⻇ CJK RADICAL HORN -> 𧢲  [U+278B2]
+    "\u2ec8": "\u8ba0",  # ⻈ CJK RADICAL C-SIMPLIFIED SPEECH -> 讠
+    "\u2ec9": "\u8d1d",  # ⻉ CJK RADICAL C-SIMPLIFIED SHELL -> 贝
+    "\u2eca": "\U00027fb7",  # ⻊ CJK RADICAL FOOT -> 𧾷  [U+27FB7]
+    "\u2ecb": "\u8f66",  # ⻋ CJK RADICAL C-SIMPLIFIED CART -> 车
+    "\u2ecc": "\u8fb6",  # ⻌ CJK RADICAL SIMPLIFIED WALK -> 辶
+    "\u2ecd": "\u8fb6",  # ⻍ CJK RADICAL WALK ONE -> 辶
+    "\u2ece": "\u8fb6",  # ⻎ CJK RADICAL WALK TWO -> 辶
+    "\u2ecf": "\u9091",  # ⻏ CJK RADICAL CITY -> 邑
+    "\u2ed0": "\u9485",  # ⻐ CJK RADICAL C-SIMPLIFIED GOLD -> 钅
+    "\u2ed1": "\u9577",  # ⻑ CJK RADICAL LONG ONE -> 長
+    "\u2ed2": "\u9578",  # ⻒ CJK RADICAL LONG TWO -> 镸
+    "\u2ed3": "\u957f",  # ⻓ CJK RADICAL C-SIMPLIFIED LONG -> 长
+    "\u2ed4": "\u95e8",  # ⻔ CJK RADICAL C-SIMPLIFIED GATE -> 门
+    "\u2ed5": "\U00028e0f",  # ⻕ CJK RADICAL MOUND ONE -> 𨸏  [U+28E0F]
+    "\u2ed6": "\u961d",  # ⻖ CJK RADICAL MOUND TWO -> 阝
+    "\u2ed7": "\u96e8",  # ⻗ CJK RADICAL RAIN -> 雨
+    "\u2ed8": "\u9752",  # ⻘ CJK RADICAL BLUE -> 青
+    "\u2ed9": "\u97e6",  # ⻙ CJK RADICAL C-SIMPLIFIED TANNED LEATHER -> 韦
+    "\u2eda": "\u9875",  # ⻚ CJK RADICAL C-SIMPLIFIED LEAF -> 页
+    "\u2edb": "\u98ce",  # ⻛ CJK RADICAL C-SIMPLIFIED WIND -> 风
+    "\u2edc": "\u98de",  # ⻜ CJK RADICAL C-SIMPLIFIED FLY -> 飞
+    "\u2edd": "\u98df",  # ⻝ CJK RADICAL EAT ONE -> 食
+    "\u2ede": "\U0002967f",  # ⻞ CJK RADICAL EAT TWO -> 𩙿  [U+2967F]
+    "\u2edf": "\u98e0",  # ⻟ CJK RADICAL EAT THREE -> 飠
+    "\u2ee0": "\u9963",  # ⻠ CJK RADICAL C-SIMPLIFIED EAT -> 饣
+    "\u2ee1": "\U00029810",  # ⻡ CJK RADICAL HEAD -> 𩠐  [U+29810]
+    "\u2ee2": "\u9a6c",  # ⻢ CJK RADICAL C-SIMPLIFIED HORSE -> 马
+    "\u2ee3": "\u9aa8",  # ⻣ CJK RADICAL BONE -> 骨
+    "\u2ee4": "\u9b3c",  # ⻤ CJK RADICAL GHOST -> 鬼
+    "\u2ee5": "\u9c7c",  # ⻥ CJK RADICAL C-SIMPLIFIED FISH -> 鱼
+    "\u2ee6": "\u9e1f",  # ⻦ CJK RADICAL C-SIMPLIFIED BIRD -> 鸟
+    "\u2ee7": "\u5364",  # ⻧ CJK RADICAL C-SIMPLIFIED SALT -> 卤
+    "\u2ee8": "\u9ea6",  # ⻨ CJK RADICAL SIMPLIFIED WHEAT -> 麦
+    "\u2ee9": "\u9ec4",  # ⻩ CJK RADICAL SIMPLIFIED YELLOW -> 黄
+    "\u2eea": "\u9efe",  # ⻪ CJK RADICAL C-SIMPLIFIED FROG -> 黾
+    "\u2eeb": "\u6589",  # ⻫ CJK RADICAL J-SIMPLIFIED EVEN -> 斉
+    "\u2eec": "\u9f50",  # ⻬ CJK RADICAL C-SIMPLIFIED EVEN -> 齐
+    "\u2eed": "\u6b6f",  # ⻭ CJK RADICAL J-SIMPLIFIED TOOTH -> 歯
+    "\u2eee": "\u9f7f",  # ⻮ CJK RADICAL C-SIMPLIFIED TOOTH -> 齿
+    "\u2eef": "\u7adc",  # ⻯ CJK RADICAL J-SIMPLIFIED DRAGON -> 竜
+    "\u2ef0": "\u9f99",  # ⻰ CJK RADICAL C-SIMPLIFIED DRAGON -> 龙
+    "\u2ef1": "\u9f9c",  # ⻱ CJK RADICAL TURTLE -> 龜
+    "\u2ef2": "\u4e80",  # ⻲ CJK RADICAL J-SIMPLIFIED TURTLE -> 亀
+    "\u2ef3": "\u9f9f",  # ⻳ CJK RADICAL C-SIMPLIFIED TURTLE -> 龟
+    "\u2f00": "\u4e00",  # ⼀ KANGXI RADICAL ONE -> 一
+    "\u2f01": "\u4e28",  # ⼁ KANGXI RADICAL LINE -> 丨
+    "\u2f02": "\u4e36",  # ⼂ KANGXI RADICAL DOT -> 丶
+    "\u2f03": "\u4e3f",  # ⼃ KANGXI RADICAL SLASH -> 丿
+    "\u2f04": "\u4e59",  # ⼄ KANGXI RADICAL SECOND -> 乙
+    "\u2f05": "\u4e85",  # ⼅ KANGXI RADICAL HOOK -> 亅
+    "\u2f06": "\u4e8c",  # ⼆ KANGXI RADICAL TWO -> 二
+    "\u2f07": "\u4ea0",  # ⼇ KANGXI RADICAL LID -> 亠
+    "\u2f08": "\u4eba",  # ⼈ KANGXI RADICAL MAN -> 人
+    "\u2f09": "\u513f",  # ⼉ KANGXI RADICAL LEGS -> 儿
+    "\u2f0a": "\u5165",  # ⼊ KANGXI RADICAL ENTER -> 入
+    "\u2f0b": "\u516b",  # ⼋ KANGXI RADICAL EIGHT -> 八
+    "\u2f0c": "\u5182",  # ⼌ KANGXI RADICAL DOWN BOX -> 冂
+    "\u2f0d": "\u5196",  # ⼍ KANGXI RADICAL COVER -> 冖
+    "\u2f0e": "\u51ab",  # ⼎ KANGXI RADICAL ICE -> 冫
+    "\u2f0f": "\u51e0",  # ⼏ KANGXI RADICAL TABLE -> 几
+    "\u2f10": "\u51f5",  # ⼐ KANGXI RADICAL OPEN BOX -> 凵
+    "\u2f11": "\u5200",  # ⼑ KANGXI RADICAL KNIFE -> 刀
+    "\u2f12": "\u529b",  # ⼒ KANGXI RADICAL POWER -> 力
+    "\u2f13": "\u52f9",  # ⼓ KANGXI RADICAL WRAP -> 勹
+    "\u2f14": "\u5315",  # ⼔ KANGXI RADICAL SPOON -> 匕
+    "\u2f15": "\u531a",  # ⼕ KANGXI RADICAL RIGHT OPEN BOX -> 匚
+    "\u2f16": "\u5338",  # ⼖ KANGXI RADICAL HIDING ENCLOSURE -> 匸
+    "\u2f17": "\u5341",  # ⼗ KANGXI RADICAL TEN -> 十
+    "\u2f18": "\u535c",  # ⼘ KANGXI RADICAL DIVINATION -> 卜
+    "\u2f19": "\u5369",  # ⼙ KANGXI RADICAL SEAL -> 卩
+    "\u2f1a": "\u5382",  # ⼚ KANGXI RADICAL CLIFF -> 厂
+    "\u2f1b": "\u53b6",  # ⼛ KANGXI RADICAL PRIVATE -> 厶
+    "\u2f1c": "\u53c8",  # ⼜ KANGXI RADICAL AGAIN -> 又
+    "\u2f1d": "\u53e3",  # ⼝ KANGXI RADICAL MOUTH -> 口
+    "\u2f1e": "\u56d7",  # ⼞ KANGXI RADICAL ENCLOSURE -> 囗
+    "\u2f1f": "\u571f",  # ⼟ KANGXI RADICAL EARTH -> 土
+    "\u2f20": "\u58eb",  # ⼠ KANGXI RADICAL SCHOLAR -> 士
+    "\u2f21": "\u5902",  # ⼡ KANGXI RADICAL GO -> 夂
+    "\u2f22": "\u590a",  # ⼢ KANGXI RADICAL GO SLOWLY -> 夊
+    "\u2f23": "\u5915",  # ⼣ KANGXI RADICAL EVENING -> 夕
+    "\u2f24": "\u5927",  # ⼤ KANGXI RADICAL BIG -> 大
+    "\u2f25": "\u5973",  # ⼥ KANGXI RADICAL WOMAN -> 女
+    "\u2f26": "\u5b50",  # ⼦ KANGXI RADICAL CHILD -> 子
+    "\u2f27": "\u5b80",  # ⼧ KANGXI RADICAL ROOF -> 宀
+    "\u2f28": "\u5bf8",  # ⼨ KANGXI RADICAL INCH -> 寸
+    "\u2f29": "\u5c0f",  # ⼩ KANGXI RADICAL SMALL -> 小
+    "\u2f2a": "\u5c22",  # ⼪ KANGXI RADICAL LAME -> 尢
+    "\u2f2b": "\u5c38",  # ⼫ KANGXI RADICAL CORPSE -> 尸
+    "\u2f2c": "\u5c6e",  # ⼬ KANGXI RADICAL SPROUT -> 屮
+    "\u2f2d": "\u5c71",  # ⼭ KANGXI RADICAL MOUNTAIN -> 山
+    "\u2f2e": "\u5ddb",  # ⼮ KANGXI RADICAL RIVER -> 巛
+    "\u2f2f": "\u5de5",  # ⼯ KANGXI RADICAL WORK -> 工
+    "\u2f30": "\u5df1",  # ⼰ KANGXI RADICAL ONESELF -> 己
+    "\u2f31": "\u5dfe",  # ⼱ KANGXI RADICAL TURBAN -> 巾
+    "\u2f32": "\u5e72",  # ⼲ KANGXI RADICAL DRY -> 干
+    "\u2f33": "\u5e7a",  # ⼳ KANGXI RADICAL SHORT THREAD -> 幺
+    "\u2f34": "\u5e7f",  # ⼴ KANGXI RADICAL DOTTED CLIFF -> 广
+    "\u2f35": "\u5ef4",  # ⼵ KANGXI RADICAL LONG STRIDE -> 廴
+    "\u2f36": "\u5efe",  # ⼶ KANGXI RADICAL TWO HANDS -> 廾
+    "\u2f37": "\u5f0b",  # ⼷ KANGXI RADICAL SHOOT -> 弋
+    "\u2f38": "\u5f13",  # ⼸ KANGXI RADICAL BOW -> 弓
+    "\u2f39": "\u5f50",  # ⼹ KANGXI RADICAL SNOUT -> 彐
+    "\u2f3a": "\u5f61",  # ⼺ KANGXI RADICAL BRISTLE -> 彡
+    "\u2f3b": "\u5f73",  # ⼻ KANGXI RADICAL STEP -> 彳
+    "\u2f3c": "\u5fc3",  # ⼼ KANGXI RADICAL HEART -> 心
+    "\u2f3d": "\u6208",  # ⼽ KANGXI RADICAL HALBERD -> 戈
+    "\u2f3e": "\u6236",  # ⼾ KANGXI RADICAL DOOR -> 戶
+    "\u2f3f": "\u624b",  # ⼿ KANGXI RADICAL HAND -> 手
+    "\u2f40": "\u652f",  # ⽀ KANGXI RADICAL BRANCH -> 支
+    "\u2f41": "\u6534",  # ⽁ KANGXI RADICAL RAP -> 攴
+    "\u2f42": "\u6587",  # ⽂ KANGXI RADICAL SCRIPT -> 文
+    "\u2f43": "\u6597",  # ⽃ KANGXI RADICAL DIPPER -> 斗
+    "\u2f44": "\u65a4",  # ⽄ KANGXI RADICAL AXE -> 斤
+    "\u2f45": "\u65b9",  # ⽅ KANGXI RADICAL SQUARE -> 方
+    "\u2f46": "\u65e0",  # ⽆ KANGXI RADICAL NOT -> 无
+    "\u2f47": "\u65e5",  # ⽇ KANGXI RADICAL SUN -> 日
+    "\u2f48": "\u66f0",  # ⽈ KANGXI RADICAL SAY -> 曰
+    "\u2f49": "\u6708",  # ⽉ KANGXI RADICAL MOON -> 月
+    "\u2f4a": "\u6728",  # ⽊ KANGXI RADICAL TREE -> 木
+    "\u2f4b": "\u6b20",  # ⽋ KANGXI RADICAL LACK -> 欠
+    "\u2f4c": "\u6b62",  # ⽌ KANGXI RADICAL STOP -> 止
+    "\u2f4d": "\u6b79",  # ⽍ KANGXI RADICAL DEATH -> 歹
+    "\u2f4e": "\u6bb3",  # ⽎ KANGXI RADICAL WEAPON -> 殳
+    "\u2f4f": "\u6bcb",  # ⽏ KANGXI RADICAL DO NOT -> 毋
+    "\u2f50": "\u6bd4",  # ⽐ KANGXI RADICAL COMPARE -> 比
+    "\u2f51": "\u6bdb",  # ⽑ KANGXI RADICAL FUR -> 毛
+    "\u2f52": "\u6c0f",  # ⽒ KANGXI RADICAL CLAN -> 氏
+    "\u2f53": "\u6c14",  # ⽓ KANGXI RADICAL STEAM -> 气
+    "\u2f54": "\u6c34",  # ⽔ KANGXI RADICAL WATER -> 水
+    "\u2f55": "\u706b",  # ⽕ KANGXI RADICAL FIRE -> 火
+    "\u2f56": "\u722a",  # ⽖ KANGXI RADICAL CLAW -> 爪
+    "\u2f57": "\u7236",  # ⽗ KANGXI RADICAL FATHER -> 父
+    "\u2f58": "\u723b",  # ⽘ KANGXI RADICAL DOUBLE X -> 爻
+    "\u2f59": "\u723f",  # ⽙ KANGXI RADICAL HALF TREE TRUNK -> 爿
+    "\u2f5a": "\u7247",  # ⽚ KANGXI RADICAL SLICE -> 片
+    "\u2f5b": "\u7259",  # ⽛ KANGXI RADICAL FANG -> 牙
+    "\u2f5c": "\u725b",  # ⽜ KANGXI RADICAL COW -> 牛
+    "\u2f5d": "\u72ac",  # ⽝ KANGXI RADICAL DOG -> 犬
+    "\u2f5e": "\u7384",  # ⽞ KANGXI RADICAL PROFOUND -> 玄
+    "\u2f5f": "\u7389",  # ⽟ KANGXI RADICAL JADE -> 玉
+    "\u2f60": "\u74dc",  # ⽠ KANGXI RADICAL MELON -> 瓜
+    "\u2f61": "\u74e6",  # ⽡ KANGXI RADICAL TILE -> 瓦
+    "\u2f62": "\u7518",  # ⽢ KANGXI RADICAL SWEET -> 甘
+    "\u2f63": "\u751f",  # ⽣ KANGXI RADICAL LIFE -> 生
+    "\u2f64": "\u7528",  # ⽤ KANGXI RADICAL USE -> 用
+    "\u2f65": "\u7530",  # ⽥ KANGXI RADICAL FIELD -> 田
+    "\u2f66": "\u758b",  # ⽦ KANGXI RADICAL BOLT OF CLOTH -> 疋
+    "\u2f67": "\u7592",  # ⽧ KANGXI RADICAL SICKNESS -> 疒
+    "\u2f68": "\u7676",  # ⽨ KANGXI RADICAL DOTTED TENT -> 癶
+    "\u2f69": "\u767d",  # ⽩ KANGXI RADICAL WHITE -> 白
+    "\u2f6a": "\u76ae",  # ⽪ KANGXI RADICAL SKIN -> 皮
+    "\u2f6b": "\u76bf",  # ⽫ KANGXI RADICAL DISH -> 皿
+    "\u2f6c": "\u76ee",  # ⽬ KANGXI RADICAL EYE -> 目
+    "\u2f6d": "\u77db",  # ⽭ KANGXI RADICAL SPEAR -> 矛
+    "\u2f6e": "\u77e2",  # ⽮ KANGXI RADICAL ARROW -> 矢
+    "\u2f6f": "\u77f3",  # ⽯ KANGXI RADICAL STONE -> 石
+    "\u2f70": "\u793a",  # ⽰ KANGXI RADICAL SPIRIT -> 示
+    "\u2f71": "\u79b8",  # ⽱ KANGXI RADICAL TRACK -> 禸
+    "\u2f72": "\u79be",  # ⽲ KANGXI RADICAL GRAIN -> 禾
+    "\u2f73": "\u7a74",  # ⽳ KANGXI RADICAL CAVE -> 穴
+    "\u2f74": "\u7acb",  # ⽴ KANGXI RADICAL STAND -> 立
+    "\u2f75": "\u7af9",  # ⽵ KANGXI RADICAL BAMBOO -> 竹
+    "\u2f76": "\u7c73",  # ⽶ KANGXI RADICAL RICE -> 米
+    "\u2f77": "\u7cf8",  # ⽷ KANGXI RADICAL SILK -> 糸
+    "\u2f78": "\u7f36",  # ⽸ KANGXI RADICAL JAR -> 缶
+    "\u2f79": "\u7f51",  # ⽹ KANGXI RADICAL NET -> 网
+    "\u2f7a": "\u7f8a",  # ⽺ KANGXI RADICAL SHEEP -> 羊
+    "\u2f7b": "\u7fbd",  # ⽻ KANGXI RADICAL FEATHER -> 羽
+    "\u2f7c": "\u8001",  # ⽼ KANGXI RADICAL OLD -> 老
+    "\u2f7d": "\u800c",  # ⽽ KANGXI RADICAL AND -> 而
+    "\u2f7e": "\u8012",  # ⽾ KANGXI RADICAL PLOW -> 耒
+    "\u2f7f": "\u8033",  # ⽿ KANGXI RADICAL EAR -> 耳
+    "\u2f80": "\u807f",  # ⾀ KANGXI RADICAL BRUSH -> 聿
+    "\u2f81": "\u8089",  # ⾁ KANGXI RADICAL MEAT -> 肉
+    "\u2f82": "\u81e3",  # ⾂ KANGXI RADICAL MINISTER -> 臣
+    "\u2f83": "\u81ea",  # ⾃ KANGXI RADICAL SELF -> 自
+    "\u2f84": "\u81f3",  # ⾄ KANGXI RADICAL ARRIVE -> 至
+    "\u2f85": "\u81fc",  # ⾅ KANGXI RADICAL MORTAR -> 臼
+    "\u2f86": "\u820c",  # ⾆ KANGXI RADICAL TONGUE -> 舌
+    "\u2f87": "\u821b",  # ⾇ KANGXI RADICAL OPPOSE -> 舛
+    "\u2f88": "\u821f",  # ⾈ KANGXI RADICAL BOAT -> 舟
+    "\u2f89": "\u826e",  # ⾉ KANGXI RADICAL STOPPING -> 艮
+    "\u2f8a": "\u8272",  # ⾊ KANGXI RADICAL COLOR -> 色
+    "\u2f8b": "\u8278",  # ⾋ KANGXI RADICAL GRASS -> 艸
+    "\u2f8c": "\u864d",  # ⾌ KANGXI RADICAL TIGER -> 虍
+    "\u2f8d": "\u866b",  # ⾍ KANGXI RADICAL INSECT -> 虫
+    "\u2f8e": "\u8840",  # ⾎ KANGXI RADICAL BLOOD -> 血
+    "\u2f8f": "\u884c",  # ⾏ KANGXI RADICAL WALK ENCLOSURE -> 行
+    "\u2f90": "\u8863",  # ⾐ KANGXI RADICAL CLOTHES -> 衣
+    "\u2f91": "\u897e",  # ⾑ KANGXI RADICAL WEST -> 襾
+    "\u2f92": "\u898b",  # ⾒ KANGXI RADICAL SEE -> 見
+    "\u2f93": "\u89d2",  # ⾓ KANGXI RADICAL HORN -> 角
+    "\u2f94": "\u8a00",  # ⾔ KANGXI RADICAL SPEECH -> 言
+    "\u2f95": "\u8c37",  # ⾕ KANGXI RADICAL VALLEY -> 谷
+    "\u2f96": "\u8c46",  # ⾖ KANGXI RADICAL BEAN -> 豆
+    "\u2f97": "\u8c55",  # ⾗ KANGXI RADICAL PIG -> 豕
+    "\u2f98": "\u8c78",  # ⾘ KANGXI RADICAL BADGER -> 豸
+    "\u2f99": "\u8c9d",  # ⾙ KANGXI RADICAL SHELL -> 貝
+    "\u2f9a": "\u8d64",  # ⾚ KANGXI RADICAL RED -> 赤
+    "\u2f9b": "\u8d70",  # ⾛ KANGXI RADICAL RUN -> 走
+    "\u2f9c": "\u8db3",  # ⾜ KANGXI RADICAL FOOT -> 足
+    "\u2f9d": "\u8eab",  # ⾝ KANGXI RADICAL BODY -> 身
+    "\u2f9e": "\u8eca",  # ⾞ KANGXI RADICAL CART -> 車
+    "\u2f9f": "\u8f9b",  # ⾟ KANGXI RADICAL BITTER -> 辛
+    "\u2fa0": "\u8fb0",  # ⾠ KANGXI RADICAL MORNING -> 辰
+    "\u2fa1": "\u8fb5",  # ⾡ KANGXI RADICAL WALK -> 辵
+    "\u2fa2": "\u9091",  # ⾢ KANGXI RADICAL CITY -> 邑
+    "\u2fa3": "\u9149",  # ⾣ KANGXI RADICAL WINE -> 酉
+    "\u2fa4": "\u91c6",  # ⾤ KANGXI RADICAL DISTINGUISH -> 釆
+    "\u2fa5": "\u91cc",  # ⾥ KANGXI RADICAL VILLAGE -> 里
+    "\u2fa6": "\u91d1",  # ⾦ KANGXI RADICAL GOLD -> 金
+    "\u2fa7": "\u9577",  # ⾧ KANGXI RADICAL LONG -> 長
+    "\u2fa8": "\u9580",  # ⾨ KANGXI RADICAL GATE -> 門
+    "\u2fa9": "\u961c",  # ⾩ KANGXI RADICAL MOUND -> 阜
+    "\u2faa": "\u96b6",  # ⾪ KANGXI RADICAL SLAVE -> 隶
+    "\u2fab": "\u96b9",  # ⾫ KANGXI RADICAL SHORT TAILED BIRD -> 隹
+    "\u2fac": "\u96e8",  # ⾬ KANGXI RADICAL RAIN -> 雨
+    "\u2fad": "\u9751",  # ⾭ KANGXI RADICAL BLUE -> 靑
+    "\u2fae": "\u975e",  # ⾮ KANGXI RADICAL WRONG -> 非
+    "\u2faf": "\u9762",  # ⾯ KANGXI RADICAL FACE -> 面
+    "\u2fb0": "\u9769",  # ⾰ KANGXI RADICAL LEATHER -> 革
+    "\u2fb1": "\u97cb",  # ⾱ KANGXI RADICAL TANNED LEATHER -> 韋
+    "\u2fb2": "\u97ed",  # ⾲ KANGXI RADICAL LEEK -> 韭
+    "\u2fb3": "\u97f3",  # ⾳ KANGXI RADICAL SOUND -> 音
+    "\u2fb4": "\u9801",  # ⾴ KANGXI RADICAL LEAF -> 頁
+    "\u2fb5": "\u98a8",  # ⾵ KANGXI RADICAL WIND -> 風
+    "\u2fb6": "\u98db",  # ⾶ KANGXI RADICAL FLY -> 飛
+    "\u2fb7": "\u98df",  # ⾷ KANGXI RADICAL EAT -> 食
+    "\u2fb8": "\u9996",  # ⾸ KANGXI RADICAL HEAD -> 首
+    "\u2fb9": "\u9999",  # ⾹ KANGXI RADICAL FRAGRANT -> 香
+    "\u2fba": "\u99ac",  # ⾺ KANGXI RADICAL HORSE -> 馬
+    "\u2fbb": "\u9aa8",  # ⾻ KANGXI RADICAL BONE -> 骨
+    "\u2fbc": "\u9ad8",  # ⾼ KANGXI RADICAL TALL -> 高
+    "\u2fbd": "\u9adf",  # ⾽ KANGXI RADICAL HAIR -> 髟
+    "\u2fbe": "\u9b25",  # ⾾ KANGXI RADICAL FIGHT -> 鬥
+    "\u2fbf": "\u9b2f",  # ⾿ KANGXI RADICAL SACRIFICIAL WINE -> 鬯
+    "\u2fc0": "\u9b32",  # ⿀ KANGXI RADICAL CAULDRON -> 鬲
+    "\u2fc1": "\u9b3c",  # ⿁ KANGXI RADICAL GHOST -> 鬼
+    "\u2fc2": "\u9b5a",  # ⿂ KANGXI RADICAL FISH -> 魚
+    "\u2fc3": "\u9ce5",  # ⿃ KANGXI RADICAL BIRD -> 鳥
+    "\u2fc4": "\u9e75",  # ⿄ KANGXI RADICAL SALT -> 鹵
+    "\u2fc5": "\u9e7f",  # ⿅ KANGXI RADICAL DEER -> 鹿
+    "\u2fc6": "\u9ea5",  # ⿆ KANGXI RADICAL WHEAT -> 麥
+    "\u2fc7": "\u9ebb",  # ⿇ KANGXI RADICAL HEMP -> 麻
+    "\u2fc8": "\u9ec3",  # ⿈ KANGXI RADICAL YELLOW -> 黃
+    "\u2fc9": "\u9ecd",  # ⿉ KANGXI RADICAL MILLET -> 黍
+    "\u2fca": "\u9ed1",  # ⿊ KANGXI RADICAL BLACK -> 黑
+    "\u2fcb": "\u9ef9",  # ⿋ KANGXI RADICAL EMBROIDERY -> 黹
+    "\u2fcc": "\u9efd",  # ⿌ KANGXI RADICAL FROG -> 黽
+    "\u2fcd": "\u9f0e",  # ⿍ KANGXI RADICAL TRIPOD -> 鼎
+    "\u2fce": "\u9f13",  # ⿎ KANGXI RADICAL DRUM -> 鼓
+    "\u2fcf": "\u9f20",  # ⿏ KANGXI RADICAL RAT -> 鼠
+    "\u2fd0": "\u9f3b",  # ⿐ KANGXI RADICAL NOSE -> 鼻
+    "\u2fd1": "\u9f4a",  # ⿑ KANGXI RADICAL EVEN -> 齊
+    "\u2fd2": "\u9f52",  # ⿒ KANGXI RADICAL TOOTH -> 齒
+    "\u2fd3": "\u9f8d",  # ⿓ KANGXI RADICAL DRAGON -> 龍
+    "\u2fd4": "\u9f9c",  # ⿔ KANGXI RADICAL TURTLE -> 龜
+    "\u2fd5": "\u9fa0",  # ⿕ KANGXI RADICAL FLUTE -> 龠
+    "\u31c6": "\U000200cc",  # ㇆ CJK STROKE HZG -> 𠃌  [U+200CC]
+    "\u31cf": "\u4e40",  # ㇏ CJK STROKE N -> 乀
+    "\u31d0": "\u4e00",  # ㇐ CJK STROKE H -> 一
+    "\u31d1": "\u4e28",  # ㇑ CJK STROKE S -> 丨
+    "\u31d2": "\u4e3f",  # ㇒ CJK STROKE P -> 丿
+    "\u31d3": "\u4e3f",  # ㇓ CJK STROKE SP -> 丿
+    "\u31d4": "\u4e36",  # ㇔ CJK STROKE D -> 丶
+    "\u31d5": "\U000200cd",  # ㇕ CJK STROKE HZ -> 𠃍  [U+200CD]
+    "\u31d6": "\u4e5b",  # ㇖ CJK STROKE HG -> 乛
+    "\u31d7": "\U000200ca",  # ㇗ CJK STROKE SZ -> 𠃊  [U+200CA]
+    "\u31d8": "\U000200ce",  # ㇘ CJK STROKE SWZ -> 𠃎  [U+200CE]
+    "\u31d9": "\U0002010c",  # ㇙ CJK STROKE ST -> 𠄌  [U+2010C]
+    "\u31da": "\u4e85",  # ㇚ CJK STROKE SG -> 亅
+    "\u31db": "\U00021fe8",  # ㇛ CJK STROKE PD -> 𡿨  [U+21FE8]
+    "\u31dc": "\U000200cb",  # ㇜ CJK STROKE PZ -> 𠃋  [U+200CB]
+    "\u31dd": "\u4e40",  # ㇝ CJK STROKE TN -> 乀
+    "\u31de": "\U000200d1",  # ㇞ CJK STROKE SZZ -> 𠃑  [U+200D1]
+    "\u31df": "\u4e5a",  # ㇟ CJK STROKE SWG -> 乚
+    "\u31e0": "\u4e59",  # ㇠ CJK STROKE HXWG -> 乙
+    "\u31e1": "\U0002010e",  # ㇡ CJK STROKE HZZZG -> 𠄎  [U+2010E]
+}

cjkfold/cli.py

@@ -0,0 +1,110 @@
+"""Command line interface for cjkfold.
+
+    cjkfold check FILE...        report look-alike characters
+    cjkfold fold  FILE...        print (or rewrite) folded text
+
+``check`` exits non-zero when it finds anything, so it can be dropped into a
+build or ingest pipeline to catch PDF extraction regressions early.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Sequence
+
+from .normalize import fold, iter_lookalikes
+
+__all__ = ["main"]
+
+
+def _read(path: Path) -> str:
+    return path.read_text(encoding="utf-8", errors="replace")
+
+
+def _cmd_check(args: argparse.Namespace) -> int:
+    total = 0
+    for name in args.paths:
+        path = Path(name)
+        text = _read(path)
+        found = list(iter_lookalikes(text))
+        total += len(found)
+        if not found:
+            if not args.quiet:
+                print(f"{path}: clean")
+            continue
+        kinds: dict[str, int] = {}
+        for item in found:
+            kinds[item.kind] = kinds.get(item.kind, 0) + 1
+        summary = ", ".join(f"{count} {kind}" for kind, count in sorted(kinds.items()))
+        print(f"{path}: {len(found)} look-alike character(s) ({summary})")
+        for item in found[: args.max] if args.max else found:
+            print(f"    {item}")
+        if args.max and len(found) > args.max:
+            print(f"    ... and {len(found) - args.max} more (raise --max to see them)")
+    if total:
+        print(f"\n{total} character(s) would change under `cjkfold fold`.")
+        return 1
+    return 0
+
+
+def _cmd_fold(args: argparse.Namespace) -> int:
+    for name in args.paths:
+        path = Path(name)
+        folded = fold(_read(path))
+        if args.write:
+            path.write_text(folded, encoding="utf-8")
+            print(f"{path}: rewritten", file=sys.stderr)
+        else:
+            if len(args.paths) > 1:
+                print(f"===== {path} =====", file=sys.stderr)
+            sys.stdout.write(folded)
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cjkfold",
+        description="Fold look-alike CJK glyphs onto their canonical ideograph.",
+    )
+    parser.add_argument("--version", action="version", version="cjkfold 0.1.0")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    check = sub.add_parser(
+        "check",
+        help="report look-alike characters (exit 1 if any are found)",
+    )
+    check.add_argument("paths", nargs="+", help="files to inspect, or - for stdin")
+    check.add_argument("--quiet", action="store_true", help="only report files with findings")
+    check.add_argument("--max", type=int, default=20, help="max findings to print per file (0 = all)")
+    check.set_defaults(func=_cmd_check)
+
+    fold_cmd = sub.add_parser("fold", help="write folded text to stdout")
+    fold_cmd.add_argument("paths", nargs="+", help="files to fold, or - for stdin")
+    fold_cmd.add_argument("-w", "--write", action="store_true", help="rewrite the files in place")
+    fold_cmd.set_defaults(func=_cmd_fold)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if getattr(args, "paths", None) and "-" in args.paths:
+        text = sys.stdin.read()
+        args.paths = [p for p in args.paths if p != "-"]
+        if args.command == "fold":
+            sys.stdout.write(fold(text))
+            return 0
+        found = list(iter_lookalikes(text))
+        for item in found[: args.max] if args.max else found:
+            print(f"     <stdin> {item}")
+        if args.max and len(found) > args.max:
+            print(f"     ... and {len(found) - args.max} more")
+        print(f"<stdin>: {len(found)} look-alike character(s)")
+        return 1 if found else 0
+    return args.func(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

cjkfold/integrations.py

@@ -0,0 +1,126 @@
+"""Framework-agnostic adapters for the two places folding has to happen.
+
+Folding belongs on *both* sides of a retrieval pipeline -- on documents as they
+go into the index, and on the query as it comes out -- and a dense score should
+not be trusted until the exact-match boost is added to it.  That is three touch
+points, and none of them is specific to any framework.
+
+These helpers therefore import nothing.  A LangChain ``Document`` carries
+``page_content``; a LlamaIndex node carries ``text``; both are ordinary
+attribute lookups, so a duck-typed adapter covers them.  Making the caller
+install a whole framework in order to fold a string would be a bad trade --
+framework APIs move, string folding does not.
+"""
+
+from __future__ import annotations
+
+import copy
+from typing import Any, Iterable, Sequence, TypeVar
+
+from .normalize import fold
+from .score import exact_boost
+
+__all__ = ["fold_documents", "rerank", "text_of"]
+
+T = TypeVar("T")
+
+# Attributes that carry the text, in priority order.  ``page_content`` is
+# LangChain, ``text`` is LlamaIndex; ``content`` catches the rest.
+_TEXT_ATTRIBUTES = ("page_content", "text", "content")
+
+
+def text_of(document: Any) -> str:
+    """Return the text carried by *document*, or raise ``TypeError``.
+
+    Recognises plain strings and objects exposing ``page_content``, ``text`` or
+    ``content`` -- which covers LangChain documents, LlamaIndex nodes, and most
+    home-grown record types.
+    """
+    if isinstance(document, str):
+        return document
+    for attribute in _TEXT_ATTRIBUTES:
+        value = getattr(document, attribute, None)
+        if isinstance(value, str):
+            return value
+    raise TypeError(
+        "cannot find text on "
+        f"{type(document).__name__}: expected a str or one of {_TEXT_ATTRIBUTES}"
+    )
+
+
+def _shallow_copy(document: Any) -> Any:
+    try:
+        return copy.copy(document)
+    except Exception:  # pragma: no cover - exotic objects without copy support
+        return document
+
+
+def fold_document(document: T, *, inplace: bool = False) -> T:
+    """Fold the text of one document.
+
+    Returns a shallow copy by default, so callers can use it in a comprehension
+    without mutating the objects they were handed.  Pass ``inplace=True`` when
+    the original is disposable.
+    """
+    target = document if inplace else _shallow_copy(document)
+
+    if isinstance(target, str):
+        return fold(target)  # type: ignore[return-value]
+
+    for attribute in _TEXT_ATTRIBUTES:
+        value = getattr(target, attribute, None)
+        if isinstance(value, str):
+            folded = fold(value)
+            if folded is not value:
+                setattr(target, attribute, folded)
+            return target
+
+    raise TypeError(
+        "cannot find text on "
+        f"{type(document).__name__}: expected a str or one of {_TEXT_ATTRIBUTES}"
+    )
+
+
+def fold_documents(documents: Iterable[T], *, inplace: bool = False) -> list[T]:
+    """Fold the text of every document -- the ingestion-side pass.
+
+    Only the text attribute is touched; ``metadata`` and everything else is left
+    exactly as it was.
+    """
+    return [fold_document(document, inplace=inplace) for document in documents]
+
+
+def rerank(
+    query: str,
+    documents: Sequence[T],
+    scores: Sequence[float],
+    *,
+    cap: float = 0.25,
+    n: int = 4,
+) -> list[tuple[T, float]]:
+    """Add the exact-match boost to *scores* and return results best-first.
+
+    This is the fix for the ranking inversion described in :mod:`cjkfold.score`:
+    a short, unrelated document out-scores a long document that contains the
+    query verbatim, purely because its embedding is tighter.  Give this the same
+    query, documents and vector scores your store returned::
+
+        hits = vectorstore.similarity_search_with_score(query, k=20)
+        docs, scores = zip(*hits)
+        for document, score in rerank(query, docs, scores):
+            ...
+
+    The query is folded internally, so it does not matter whether it was typed
+    with ordinary characters or with look-alikes.
+    """
+    if len(documents) != len(scores):
+        raise ValueError(
+            f"documents and scores must be the same length "
+            f"(got {len(documents)} and {len(scores)})"
+        )
+    combined = [
+        (document, float(score) + exact_boost(query, text_of(document), n=n, cap=cap))
+        for document, score in zip(documents, scores)
+    ]
+    combined.sort(key=lambda pair: pair[1], reverse=True)
+    return combined

cjkfold/normalize.py

@@ -0,0 +1,138 @@
+"""Fold look-alike CJK glyphs onto their canonical ideograph.
+
+Text extracted from PDFs -- and from some OCR pipelines -- regularly contains
+characters that *render* identically to an ordinary Han character while sitting
+on a different code point::
+
+    ⼀  U+2F00  KANGXI RADICAL ONE           renders as  一  U+4E00
+    ⻆  U+2EC6  CJK RADICAL SIMPLIFIED HORN  renders as  角  U+89D2
+
+Nothing raises.  Comparison simply starts disagreeing::
+
+    >>> "紫色" in extracted_text
+    False
+
+and a retriever quietly ranks an unrelated short document above the one that
+contains the query verbatim.
+
+NFKC already solves the Kangxi block (U+2F00-U+2FDF): all 214 of its characters
+carry compatibility decompositions.  It does *not* solve the CJK Radicals
+Supplement (U+2E80-U+2EFF), where 113 of the 115 named characters survive NFKC
+untouched -- Unicode withholds compatibility mappings there deliberately, since
+those glyphs are distinct encoded radicals rather than compatibility variants.
+
+This module closes that gap using the ``EquivalentUnifiedIdeograph`` property
+from the Unicode Character Database (Unicode 15.1+).
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from dataclasses import dataclass
+from typing import Iterator
+
+from ._table import EQUIVALENT_IDEOGRAPH
+
+__all__ = [
+    "fold",
+    "has_lookalikes",
+    "iter_lookalikes",
+    "Lookalike",
+    "kind_of",
+]
+
+# str.translate() mapping: code point -> canonical form
+_FOLD: dict[int, str] = {ord(char): target for char, target in EQUIVALENT_IDEOGRAPH.items()}
+
+# Radic supplement: the block NFKC cannot repair.
+_RADICAL_SUPPLEMENT = (0x2E80, 0x2EFF)
+# Kangxi radicals: NFKC repairs these, included for completeness / older engines.
+_KANGXI = (0x2F00, 0x2FDF)
+# CJK strokes: stroke glyphs that fold onto a look-alike ideograph.
+_CJK_STROKE = (0x31C0, 0x31EF)
+
+
+def kind_of(codepoint: int) -> str:
+    """Classify a code point by the Unicode block that causes the look-alike."""
+    if _RADICAL_SUPPLEMENT[0] <= codepoint <= _RADICAL_SUPPLEMENT[1]:
+        return "radical-supplement"
+    if _KANGXI[0] <= codepoint <= _KANGXI[1]:
+        return "kangxi"
+    if _CJK_STROKE[0] <= codepoint <= _CJK_STROKE[1]:
+        return "cjk-stroke"
+    return "other"
+
+
+@dataclass(frozen=True)
+class Lookalike:
+    """One character that renders like a different, more common character."""
+
+    index: int
+    line: int
+    column: int
+    char: str
+    codepoint: int
+    replacement: str
+    name: str
+    kind: str
+
+    @property
+    def codepoint_label(self) -> str:
+        return f"U+{self.codepoint:04X}"
+
+    @property
+    def replacement_label(self) -> str:
+        return f"U+{ord(self.replacement):04X}"
+
+    def __str__(self) -> str:  # pragma: no cover - trivial
+        return (
+            f"{self.line}:{self.column}  {self.char} {self.codepoint_label} "
+            f"-> {self.replacement} {self.replacement_label}  ({self.name})"
+        )
+
+
+def fold(text: str) -> str:
+    """Return *text* with every look-alike glyph replaced by its canonical form.
+
+    Purely a character-level fold: it never rewrites full-width punctuation,
+    ligatures or other NFKC compatibility characters, so it is safe to apply to
+    text that will still be displayed or stored.  Use
+    ``unicodedata.normalize("NFKC", ...)`` as well if you also want those.
+
+    Idempotent -- ``fold(fold(s)) == fold(s)`` -- and returns the original
+    string object unchanged when there is nothing to fold.
+    """
+    if not text:
+        return text
+    for char in text:
+        if ord(char) in _FOLD:
+            return text.translate(_FOLD)
+    return text
+
+
+def has_lookalikes(text: str) -> bool:
+    """True if *text* contains at least one foldable code point."""
+    return any(ord(char) in _FOLD for char in text)
+
+
+def iter_lookalikes(text: str) -> Iterator[Lookalike]:
+    """Yield every foldable character in *text*, in order, with 1-based position."""
+    line, column = 1, 1
+    for index, char in enumerate(text):
+        codepoint = ord(char)
+        replacement = _FOLD.get(codepoint)
+        if replacement is not None:
+            yield Lookalike(
+                index=index,
+                line=line,
+                column=column,
+                char=char,
+                codepoint=codepoint,
+                replacement=replacement,
+                name=unicodedata.name(char, "<unnamed>"),
+                kind=kind_of(codepoint),
+            )
+        if char == "\n":
+            line, column = line + 1, 1
+        else:
+            column += 1

cjkfold/score.py

@@ -0,0 +1,87 @@
+"""Score bonuses that restore exact-token ranking in dense retrieval.
+
+Embeddings are bad at literal tokens.  A rare identifier or short code sits
+almost nowhere useful in vector space, and a *short*, unrelated document can
+easily out-score the long document that contains the token verbatim -- short
+documents have tighter embeddings and therefore inflated cosine similarity.
+
+Measured on a real corpus, the document that literally contained the query
+string scored 0.29 while an unrelated short document scored 0.36: the ranking
+was simply inverted.
+
+:func:`exact_boost` restores the ordering by rewarding character n-grams of the
+query that occur verbatim in the document.  Add the result to your vector score
+and re-rank.
+
+Both sides are folded through :func:`cjkfold.fold` internally.  Folding only the
+index, or only the query, silently disables the whole mechanism -- so the
+helper does it for you and the API gives you no way to get it wrong.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from .normalize import fold
+
+__all__ = ["ngram_overlap", "exact_boost", "ngrams"]
+
+
+def ngrams(text: str, n: int = 4) -> set[str]:
+    """Return the set of character *n*-grams of *text*.
+
+    Text shorter than *n* yields the whole string (or nothing, if empty), so a
+    two-character query still produces one comparison unit.
+    """
+    if n < 1:
+        raise ValueError("n must be >= 1")
+    folded = fold(text)
+    if not folded:
+        return set()
+    if len(folded) <= n:
+        return {folded}
+    return {folded[i : i + n] for i in range(len(folded) - n + 1)}
+
+
+def ngram_overlap(query: str, document: str, *, n: int = 4) -> float:
+    """Fraction of the query's n-grams that appear verbatim in *document* (0..1).
+
+    Queries shorter than *n* degrade to a containment test, because a query of
+    two or three characters yields no shingle to compare.  Without that branch
+    every short query scores 0.0 -- the mechanism silently does nothing for
+    exactly the queries users type most.
+    """
+    query_folded = fold(query)
+    if not query_folded:
+        return 0.0
+    document_folded = fold(document)
+
+    if len(query_folded) <= n:
+        return 1.0 if query_folded in document_folded else 0.0
+
+    query_grams = ngrams(query_folded, n)
+    if not query_grams:
+        return 0.0
+    document_grams = ngrams(document_folded, n)
+    return len(query_grams & document_grams) / len(query_grams)
+
+
+def exact_boost(
+    query: str,
+    document: str,
+    *,
+    n: int = 4,
+    cap: float = 0.25,
+) -> float:
+    """Bonus in ``[0, cap]`` for literal n-gram overlap between query and document.
+
+    Scales linearly with :func:`ngram_overlap`, so a document containing the
+    entire query earns the full ``cap`` while one sharing half of it earns half.
+
+    Typical use::
+
+        score = vector_similarity(query, document) + exact_boost(query, document)
+    """
+    if cap < 0:
+        raise ValueError("cap must be >= 0")
+    return cap * ngram_overlap(query, document, n=n)

cjkfold-0.1.0.dist-info/METADATA

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: cjkfold
+Version: 0.1.0
+Summary: Fold look-alike CJK glyphs (Kangxi radicals, CJK radical supplements) so text extracted from PDFs matches again.
+Author: cdcailf
+License: MIT
+Project-URL: Homepage, https://github.com/cdcailf/cjkfold
+Project-URL: Repository, https://github.com/cdcailf/cjkfold
+Project-URL: Issues, https://github.com/cdcailf/cjkfold/issues
+Keywords: cjk,chinese,unicode,nfkc,normalization,pdf-extraction,rag,retrieval,text-matching
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: Chinese (Simplified)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# cjkfold
+
+Text pulled out of a PDF often contains characters that **look** like ordinary
+Han characters but are not them. Comparison against them quietly fails.
+
+```
+⻆  U+2EC6  CJK RADICAL SIMPLIFIED HORN   renders as  角  U+89D2
+⾊  U+2F8A  KANGXI RADICAL COLOR          renders as  色  U+8272
+⺮  U+2EAE  CJK RADICAL BAMBOO            folds to    𥫗  U+25AD7
+```
+
+`cjkfold` folds them back onto the characters you meant:
+
+```python
+from cjkfold import fold
+
+fold("标准颜⾊值与⻆色按钮")
+# '标准颜色值与角色按钮'
+
+"颜色" in "标准颜⾊值"          # False  ← the bug
+"颜色" in fold("标准颜⾊值")     # True
+```
+
+Zero runtime dependencies. Pure functions. Python 3.9+.
+
+---
+
+**背景阅读 / Background:** [PDF 抽出来的中文，为什么搜不到](docs/why-cjk-text-breaks.md)
+—— 完整的排查经过（中文），包括我自己踩的四个坑。
+社区版：[掘金](https://juejin.cn/post/7695671592383807542)
+
+**Built on this:** [corpusfold](https://github.com/cdcailf/corpusfold) — this
+character layer wired into a full PDF → knowledge-base pipeline (抽取 / 分栏 /
+表格 / 分块 / 混合检索).
+
+## The problem
+
+Nothing raises. The text displays correctly, because the glyphs really are
+identical. What breaks is everything downstream that compares characters:
+
+- `"紫色" in text` returns `False` for a document that shows 紫色
+- a keyword search misses
+- a vector retriever ranks an unrelated *short* document above the one that
+  contains the query verbatim, because a short document's embedding is tighter
+  and scores a higher cosine — measured on a real corpus, the relevant document
+  scored **0.29** against a decoy's **0.36**
+- deduplication, caching and ground-truth checks all start disagreeing
+
+## Why NFKC is not enough
+
+The reflex is `unicodedata.normalize("NFKC", text)`. It handles the **Kangxi
+Radicals** block completely — all 214 characters carry compatibility
+decompositions — and then stops:
+
+| Block | Characters | Repaired by NFKC | Repaired by cjkfold |
+|---|---|---|---|
+| Kangxi Radicals `U+2F00–U+2FDF` | 214 | **214** | 214 |
+| CJK Radicals Supplement `U+2E80–U+2EFF` | 115 | **2** | **114** |
+
+Unicode withholds compatibility mappings in the supplement block *on purpose*:
+those glyphs are distinct encoded radicals, not compatibility variants of an
+existing character. So NFKC leaves 113 of the 115 standing.
+
+`cjkfold` closes that gap with the `EquivalentUnifiedIdeograph` property from
+the Unicode Character Database (Unicode 15.1+), which maps 114 of them onto a
+unified ideograph. The one remainder, `⺀ U+2E80 CJK RADICAL REPEAT`, has no
+ideograph equivalent and is left alone.
+
+## Install
+
+Not published on PyPI yet, so install from a checkout:
+
+```bash
+git clone https://github.com/cdcailf/cjkfold
+cd cjkfold
+uv venv .venv --python 3.11
+uv pip install -e .
+```
+
+Once it is published this collapses to `pip install cjkfold`.
+
+## Usage
+
+```python
+from cjkfold import fold, has_lookalikes, iter_lookalikes
+
+fold("紫⾊的⻆落")            # '紫色的角落'
+fold("普通文本")              # unchanged; returns the same object
+
+has_lookalikes("⻆")           # True
+has_lookalikes("角")           # False
+
+for item in iter_lookalikes("第一行\n第⼆行⻆"):
+    print(item)
+# 2:2  ⼆ U+2F06 -> 二 U+4E8C  (KANGXI RADICAL TWO)
+# 2:4  ⻆ U+2EC6 -> 角 U+89D2  (CJK RADICAL SIMPLIFIED HORN)
+```
+
+`fold()` is idempotent, and it is deliberately **not** an NFKC pass: it will
+not touch full-width punctuation (`，` `；` `（` `）`), compatibility characters
+or ligatures, so it is safe to apply to text that is still going to be stored or
+displayed. Combine with NFKC yourself if you want both:
+
+```python
+import unicodedata
+clean = fold(unicodedata.normalize("NFKC", text))
+```
+
+### Command line
+
+```bash
+cjkfold check document.txt      # report look-alikes, exit 1 if any (CI-friendly)
+cjkfold check --quiet docs/*.txt
+cjkfold fold document.txt       # print the folded text
+cjkfold fold -w document.txt    # rewrite in place
+cat document.txt | cjkfold check -
+```
+
+```
+$ cjkfold check extracted.txt
+extracted.txt: 3 look-alike character(s) (1 kangxi, 2 radical-supplement)
+    1:4  ⾊ U+2F8A -> 色 U+8272  (KANGXI RADICAL COLOR)
+    1:7  ⻆ U+2EC6 -> 角 U+89D2  (CJK RADICAL SIMPLIFIED HORN)
+    2:5  ⺮ U+2EAE -> 𥫗 U+25AD7  (CJK RADICAL BAMBOO)
+```
+
+Drop `cjkfold check` into an ingest pipeline to catch extraction regressions
+before they reach the index.
+
+## Retrieval helper
+
+Folding is the fix; ranking is the follow-up. Dense retrieval is bad at literal
+tokens, so `exact_boost` adds a bonus for query n-grams that appear verbatim in a
+document:
+
+```python
+from cjkfold import exact_boost
+
+score = vector_similarity(query, document) + exact_boost(query, document)
+```
+
+Both sides are folded inside the helper. That is not a convenience — folding the
+index but not the query (or the reverse) silently disables the whole mechanism,
+because nothing ever matches. The API gives you no way to get it wrong.
+
+Queries shorter than the shingle size fall back to a containment test, so a
+two-character query is not silently scored zero.
+
+## Wiring it into a pipeline
+
+Three touch points, zero framework dependencies:
+
+```python
+from cjkfold import fold, fold_documents, rerank
+
+# 1. on the way in -- clean the corpus before it is embedded
+documents = fold_documents(loader.load())
+
+# 2. on the way out -- fold the query so it can match
+hits = vectorstore.similarity_search_with_score(fold(question), k=20)
+
+# 3. before you trust the order -- restore the verbatim matches
+documents, scores = zip(*hits)
+for document, score in rerank(question, documents, scores):
+    ...
+```
+
+`fold_documents` and `rerank` recognise LangChain documents (`page_content`),
+LlamaIndex nodes (`text`) and plain strings -- and import none of them. Making
+you install a framework in order to fold a string would be a bad trade:
+framework APIs move, string folding does not.
+
+## Caveats
+
+- **Glyph folding only.** No traditional⇄simplified conversion, no z-variant
+  normalisation. Those are different problems with different trade-offs.
+- **Table follows the latest UCD**, which can run ahead of the `unicodedata`
+  database bundled with an older Python — 3.9 ships Unicode 13, where one target
+  (`U+2B739`) is not yet assigned. Folding is unaffected (it is a plain code point
+  substitution), but `unicodedata.name()` on that result will come up empty.
+- **Existing stored text is not retroactively repaired.** Fold on write; if you
+  have already indexed unfolded text, re-ingest or re-fold it.
+- Characters above the BMP are written as `\U000xxxxx` in the table. A `\uXXXX`
+  escape silently truncates there and turns one character into two — the table
+  test suite guards against exactly that regression.
+
+## Regenerating the table
+
+`src/cjkfold/_table.py` is generated, never hand-edited:
+
+```bash
+python scripts/build_table.py                  # fetch from unicode.org
+python scripts/build_table.py CJKRadicals.txt  # or use a local copy
+```
+
+## Development
+
+```bash
+uv venv .venv --python 3.9
+uv pip install -e ".[dev]"
+.venv/bin/python -m pytest
+```
+
+## 中文简述
+
+PDF 抽出来的中文常常混着**形近异码**字符：`⻆` 和 `角` 长得一样、码位不同，于是
+`"颜色" in 文本` 返回 `False`，检索排错、去重失效、缓存打不中。
+
+`unicodedata.normalize("NFKC", ...)` 只能解决康熙部首区（214 个全都能还原），
+**解决不了 CJK 部首补充区**——那个区 115 个字符里 NFKC 只救得了 2 个。
+
+`cjkfold` 用 Unicode 官方的 `EquivalentUnifiedIdeograph` 属性补上这个缺口，
+115 个里能还原 114 个。同时提供 `exact_boost()`，把"查询片段在文档里逐字命中"
+补进向量相似度，避免短文档靠余弦虚高把真正命中的长文档挤下去。
+
+## License
+
+MIT

cjkfold-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+cjkfold/__init__.py,sha256=_EOUkgQR8spIOiVIGB0wRWUyCKNI2EpUx-5dEmoGhJg,968
+cjkfold/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+cjkfold/_table.py,sha256=4dDN1c7igxcWOcwAbP7-bG1g_OuCtJO_DCdGvKPHRVo,21548
+cjkfold/cli.py,sha256=vJS3iK7ZwBOGKQeziN5RH9xCDHYvdx5LTTPulXZyv3g,3923
+cjkfold/integrations.py,sha256=NMZzDJzTHwF49aEcJW9BKiOvLnblfxV724UhPJkJauw,4487
+cjkfold/normalize.py,sha256=Fd5VTWeaFqPVZPHhUdoitRc6UoA22etm_jTLUAhjI_A,4526
+cjkfold/score.py,sha256=zVICOIjegNUGz_2kH8lwsBF0lCDqPU1lhIc5DkPW0iM,3015
+cjkfold-0.1.0.dist-info/licenses/LICENSE,sha256=ocrzPN24a0cu0LMN4cam137idMPpCS_nP8Iex87K2MM,1064
+cjkfold-0.1.0.dist-info/METADATA,sha256=QlTnPgx9-Xnrnak3Vr7-2M8iNyyeXjnj3Qs49mAIOTo,9130
+cjkfold-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+cjkfold-0.1.0.dist-info/entry_points.txt,sha256=H1ubhADv91yVhByDxooDt1xHX_caN-XEEy3qw9JZXyE,45
+cjkfold-0.1.0.dist-info/top_level.txt,sha256=dxf3JC3zYsVaUv9MGR-V0xaF6JnPC8LAJI_TghpK5d8,8
+cjkfold-0.1.0.dist-info/RECORD,,

cjkfold-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cjkfold-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cjkfold = cjkfold.cli:main

cjkfold-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 cdcailf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cjkfold-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cjkfold