zilib 0.1.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zilib-0.1.9/.gitignore +12 -0
- zilib-0.1.9/.lastmake +0 -0
- zilib-0.1.9/Cargo.lock +232 -0
- zilib-0.1.9/Cargo.toml +18 -0
- zilib-0.1.9/LICENSE.md +18 -0
- zilib-0.1.9/Makefile +33 -0
- zilib-0.1.9/NOTES.md +7 -0
- zilib-0.1.9/PKG-INFO +18 -0
- zilib-0.1.9/README.md +80 -0
- zilib-0.1.9/ad_hoc_tests/Makefile +4 -0
- zilib-0.1.9/ad_hoc_tests/README.md +1 -0
- zilib-0.1.9/ad_hoc_tests/test_binary_search.sh +20 -0
- zilib-0.1.9/ad_hoc_tests/test_binary_search_fast.sh +525 -0
- zilib-0.1.9/lists/CJKRadicals.txt +283 -0
- zilib-0.1.9/lists/HKVariants.txt +63 -0
- zilib-0.1.9/lists/TWVariants.txt +39 -0
- zilib-0.1.9/lists/charlist.json +1 -0
- zilib-0.1.9/lists/edb_charlist.txt +558 -0
- zilib-0.1.9/lists/en_unigram_freq.csv.bz2 +0 -0
- zilib-0.1.9/lists/english_variants.json +2792 -0
- zilib-0.1.9/lists/hfhchan-kVariants.txt +10629 -0
- zilib-0.1.9/lists/index.md +242 -0
- zilib-0.1.9/lists/varcon.txt.bz2 +0 -0
- zilib-0.1.9/lists/wordshk_autoconvert.json +46 -0
- zilib-0.1.9/lists/wordshk_autoconvert.txt +100 -0
- zilib-0.1.9/lists/wordshk_charlist.txt +179 -0
- zilib-0.1.9/lists/wordshk_charset.json +213 -0
- zilib-0.1.9/lists/wordshk_variantmap.json +206 -0
- zilib-0.1.9/lists/wordshk_variantmap.txt +150 -0
- zilib-0.1.9/lists/wordslist.csv +59871 -0
- zilib-0.1.9/pyproject.toml +24 -0
- zilib-0.1.9/python/zilib/__init__.py +56 -0
- zilib-0.1.9/python/zilib/lists/CJKRadicals.txt +283 -0
- zilib-0.1.9/python/zilib/lists/HKVariants.txt +63 -0
- zilib-0.1.9/python/zilib/lists/TWVariants.txt +39 -0
- zilib-0.1.9/python/zilib/lists/charlist.json +1 -0
- zilib-0.1.9/python/zilib/lists/edb_charlist.txt +558 -0
- zilib-0.1.9/python/zilib/lists/english_variants.json +2792 -0
- zilib-0.1.9/python/zilib/lists/hfhchan-kVariants.txt +10629 -0
- zilib-0.1.9/python/zilib/lists/index.md +242 -0
- zilib-0.1.9/python/zilib/lists/wordshk_autoconvert.json +46 -0
- zilib-0.1.9/python/zilib/lists/wordshk_autoconvert.txt +100 -0
- zilib-0.1.9/python/zilib/lists/wordshk_charlist.txt +179 -0
- zilib-0.1.9/python/zilib/lists/wordshk_charset.json +213 -0
- zilib-0.1.9/python/zilib/lists/wordshk_variantmap.json +206 -0
- zilib-0.1.9/python/zilib/lists/wordshk_variantmap.txt +150 -0
- zilib-0.1.9/python/zilib/lists/wordslist.csv +59871 -0
- zilib-0.1.9/scripts/bump.py +47 -0
- zilib-0.1.9/src/bin/binary_search_file.rs +71 -0
- zilib-0.1.9/src/bin/guess_jyutping.rs +15 -0
- zilib-0.1.9/src/bin/radical_sort.rs +54 -0
- zilib-0.1.9/src/bin/zicount.rs +47 -0
- zilib-0.1.9/src/bin/zigen.rs +443 -0
- zilib-0.1.9/src/cantonese.rs +73 -0
- zilib-0.1.9/src/cjk.rs +111 -0
- zilib-0.1.9/src/common.rs +247 -0
- zilib-0.1.9/src/data.rs +361 -0
- zilib-0.1.9/src/english.rs +456 -0
- zilib-0.1.9/src/lib.rs +8 -0
- zilib-0.1.9/src/ruby_match.rs +371 -0
- zilib-0.1.9/src/segmentation.rs +158 -0
- zilib-0.1.9/zilib-python/.cargo/config +12 -0
- zilib-0.1.9/zilib-python/Cargo.lock +463 -0
- zilib-0.1.9/zilib-python/Cargo.toml +16 -0
- zilib-0.1.9/zilib-python/Makefile +11 -0
- zilib-0.1.9/zilib-python/README.md +3 -0
- zilib-0.1.9/zilib-python/copy_function_definitions.py +106 -0
- zilib-0.1.9/zilib-python/copy_packaged_lists +36 -0
- zilib-0.1.9/zilib-python/python/zilib/__init__.py +56 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/CJKRadicals.txt +283 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/HKVariants.txt +63 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/TWVariants.txt +39 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/charlist.json +1 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/edb_charlist.txt +558 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/english_variants.json +2792 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/hfhchan-kVariants.txt +10629 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/index.md +242 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_autoconvert.json +46 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_autoconvert.txt +100 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_charlist.txt +179 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_charset.json +213 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_variantmap.json +206 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_variantmap.txt +150 -0
- zilib-0.1.9/zilib-python/python/zilib/lists/wordslist.csv +59871 -0
- zilib-0.1.9/zilib-python/src/lib.rs +189 -0
- zilib-0.1.9/zilib-python/tests/test_general.py +63 -0
- zilib-0.1.9/zilib-python/tests/test_wordshk_original.py +423 -0
zilib-0.1.9/.gitignore
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/target
|
|
2
|
+
/python/target
|
|
3
|
+
/venv
|
|
4
|
+
/lists/Unihan.zip
|
|
5
|
+
/lists/Unihan_DictionaryIndices.txt
|
|
6
|
+
/lists/Unihan_DictionaryLikeData.txt
|
|
7
|
+
/lists/Unihan_IRGSources.txt
|
|
8
|
+
/lists/Unihan_NumericValues.txt
|
|
9
|
+
/lists/Unihan_OtherMappings.txt
|
|
10
|
+
/lists/Unihan_RadicalStrokeCounts.txt
|
|
11
|
+
/lists/Unihan_Readings.txt
|
|
12
|
+
/lists/Unihan_Variants.txt
|
zilib-0.1.9/.lastmake
ADDED
|
File without changes
|
zilib-0.1.9/Cargo.lock
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 3
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.2"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "bzip2"
|
|
16
|
+
version = "0.4.4"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"bzip2-sys",
|
|
21
|
+
"libc",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[[package]]
|
|
25
|
+
name = "bzip2-sys"
|
|
26
|
+
version = "0.1.11+1.0.8"
|
|
27
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
28
|
+
checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"cc",
|
|
31
|
+
"libc",
|
|
32
|
+
"pkg-config",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[[package]]
|
|
36
|
+
name = "cc"
|
|
37
|
+
version = "1.0.83"
|
|
38
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
39
|
+
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
|
|
40
|
+
dependencies = [
|
|
41
|
+
"libc",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[[package]]
|
|
45
|
+
name = "cfg-if"
|
|
46
|
+
version = "1.0.0"
|
|
47
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
48
|
+
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
|
49
|
+
|
|
50
|
+
[[package]]
|
|
51
|
+
name = "csv"
|
|
52
|
+
version = "1.3.0"
|
|
53
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
54
|
+
checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
|
|
55
|
+
dependencies = [
|
|
56
|
+
"csv-core",
|
|
57
|
+
"itoa",
|
|
58
|
+
"ryu",
|
|
59
|
+
"serde",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[[package]]
|
|
63
|
+
name = "csv-core"
|
|
64
|
+
version = "0.1.11"
|
|
65
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
66
|
+
checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
|
|
67
|
+
dependencies = [
|
|
68
|
+
"memchr",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[[package]]
|
|
72
|
+
name = "encoding_rs"
|
|
73
|
+
version = "0.8.33"
|
|
74
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
75
|
+
checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
|
|
76
|
+
dependencies = [
|
|
77
|
+
"cfg-if",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
[[package]]
|
|
81
|
+
name = "encoding_rs_io"
|
|
82
|
+
version = "0.1.7"
|
|
83
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
|
+
checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
|
|
85
|
+
dependencies = [
|
|
86
|
+
"encoding_rs",
|
|
87
|
+
]
|
|
88
|
+
|
|
89
|
+
[[package]]
|
|
90
|
+
name = "itoa"
|
|
91
|
+
version = "1.0.10"
|
|
92
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
93
|
+
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
|
|
94
|
+
|
|
95
|
+
[[package]]
|
|
96
|
+
name = "libc"
|
|
97
|
+
version = "0.2.153"
|
|
98
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
99
|
+
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
|
|
100
|
+
|
|
101
|
+
[[package]]
|
|
102
|
+
name = "memchr"
|
|
103
|
+
version = "2.7.1"
|
|
104
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
105
|
+
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
|
|
106
|
+
|
|
107
|
+
[[package]]
|
|
108
|
+
name = "pkg-config"
|
|
109
|
+
version = "0.3.29"
|
|
110
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
111
|
+
checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb"
|
|
112
|
+
|
|
113
|
+
[[package]]
|
|
114
|
+
name = "proc-macro2"
|
|
115
|
+
version = "1.0.78"
|
|
116
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
117
|
+
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
|
|
118
|
+
dependencies = [
|
|
119
|
+
"unicode-ident",
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
[[package]]
|
|
123
|
+
name = "quote"
|
|
124
|
+
version = "1.0.35"
|
|
125
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
126
|
+
checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
|
|
127
|
+
dependencies = [
|
|
128
|
+
"proc-macro2",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
[[package]]
|
|
132
|
+
name = "regex"
|
|
133
|
+
version = "1.10.3"
|
|
134
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
135
|
+
checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
|
|
136
|
+
dependencies = [
|
|
137
|
+
"aho-corasick",
|
|
138
|
+
"memchr",
|
|
139
|
+
"regex-automata",
|
|
140
|
+
"regex-syntax",
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
[[package]]
|
|
144
|
+
name = "regex-automata"
|
|
145
|
+
version = "0.4.5"
|
|
146
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
147
|
+
checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
|
|
148
|
+
dependencies = [
|
|
149
|
+
"aho-corasick",
|
|
150
|
+
"memchr",
|
|
151
|
+
"regex-syntax",
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
[[package]]
|
|
155
|
+
name = "regex-syntax"
|
|
156
|
+
version = "0.8.2"
|
|
157
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
158
|
+
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
|
|
159
|
+
|
|
160
|
+
[[package]]
|
|
161
|
+
name = "ryu"
|
|
162
|
+
version = "1.0.16"
|
|
163
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
164
|
+
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
|
|
165
|
+
|
|
166
|
+
[[package]]
|
|
167
|
+
name = "serde"
|
|
168
|
+
version = "1.0.196"
|
|
169
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
170
|
+
checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
|
|
171
|
+
dependencies = [
|
|
172
|
+
"serde_derive",
|
|
173
|
+
]
|
|
174
|
+
|
|
175
|
+
[[package]]
|
|
176
|
+
name = "serde_derive"
|
|
177
|
+
version = "1.0.196"
|
|
178
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
179
|
+
checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
|
|
180
|
+
dependencies = [
|
|
181
|
+
"proc-macro2",
|
|
182
|
+
"quote",
|
|
183
|
+
"syn",
|
|
184
|
+
]
|
|
185
|
+
|
|
186
|
+
[[package]]
|
|
187
|
+
name = "serde_json"
|
|
188
|
+
version = "1.0.113"
|
|
189
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
190
|
+
checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
|
|
191
|
+
dependencies = [
|
|
192
|
+
"itoa",
|
|
193
|
+
"ryu",
|
|
194
|
+
"serde",
|
|
195
|
+
]
|
|
196
|
+
|
|
197
|
+
[[package]]
|
|
198
|
+
name = "syn"
|
|
199
|
+
version = "2.0.48"
|
|
200
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
201
|
+
checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
|
|
202
|
+
dependencies = [
|
|
203
|
+
"proc-macro2",
|
|
204
|
+
"quote",
|
|
205
|
+
"unicode-ident",
|
|
206
|
+
]
|
|
207
|
+
|
|
208
|
+
[[package]]
|
|
209
|
+
name = "unicode-ident"
|
|
210
|
+
version = "1.0.12"
|
|
211
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
212
|
+
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
|
213
|
+
|
|
214
|
+
[[package]]
|
|
215
|
+
name = "unicode_categories"
|
|
216
|
+
version = "0.1.1"
|
|
217
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
218
|
+
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
|
|
219
|
+
|
|
220
|
+
[[package]]
|
|
221
|
+
name = "zilib"
|
|
222
|
+
version = "0.1.9"
|
|
223
|
+
dependencies = [
|
|
224
|
+
"bzip2",
|
|
225
|
+
"csv",
|
|
226
|
+
"encoding_rs",
|
|
227
|
+
"encoding_rs_io",
|
|
228
|
+
"regex",
|
|
229
|
+
"serde",
|
|
230
|
+
"serde_json",
|
|
231
|
+
"unicode_categories",
|
|
232
|
+
]
|
zilib-0.1.9/Cargo.toml
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "zilib"
|
|
3
|
+
version = "0.1.9"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
|
|
6
|
+
[dependencies]
|
|
7
|
+
bzip2 = "0.4.4"
|
|
8
|
+
csv = "1.3.0"
|
|
9
|
+
encoding_rs = "0.8.33"
|
|
10
|
+
encoding_rs_io = "0.1.7"
|
|
11
|
+
regex = "1.10.3"
|
|
12
|
+
serde = { version = "1.0.196", features = ["derive"] }
|
|
13
|
+
serde_json = "1.0.113"
|
|
14
|
+
unicode_categories = "0.1.1"
|
|
15
|
+
|
|
16
|
+
[features]
|
|
17
|
+
default = []
|
|
18
|
+
downloaded_data = []
|
zilib-0.1.9/LICENSE.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
## Overview
|
|
2
|
+
|
|
3
|
+
- Code in this repository is released under the following license unless otherwise stated.
|
|
4
|
+
|
|
5
|
+
- For information on data files, see [lists/index.md](lists/index.md) instead.
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
## License
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
Copyright 2024 Hong Kong Lexicography Limited.
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
14
|
+
|
|
15
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
16
|
+
|
|
17
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
18
|
+
```
|
zilib-0.1.9/Makefile
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
all: zigen_data rust_all
|
|
2
|
+
rust_all: zigen_data lists/wordslist.csv
|
|
3
|
+
cargo build --lib
|
|
4
|
+
lint:
|
|
5
|
+
cargo clippy
|
|
6
|
+
|
|
7
|
+
target/release/%: src/*.rs src/bin/*.rs lists/wordslist.csv
|
|
8
|
+
cargo build --release --bins --no-default-features --features downloaded_data
|
|
9
|
+
|
|
10
|
+
zigen_data: lists/Unihan.zip lists/CJKRadicals.txt lists/english_variants.json lists/wordshk_charset.json lists/wordshk_variantmap.json lists/wordshk_autoconvert.json
|
|
11
|
+
|
|
12
|
+
lists/wordslist.csv:
|
|
13
|
+
cd lists && curl -O https://words.hk/faiman/analysis/wordslist.csv
|
|
14
|
+
lists/Unihan.zip:
|
|
15
|
+
cd lists && curl -O https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip && unzip Unihan.zip
|
|
16
|
+
lists/CJKRadicals.txt:
|
|
17
|
+
cd lists && curl -O https://www.unicode.org/Public/UCD/latest/ucd/CJKRadicals.txt
|
|
18
|
+
|
|
19
|
+
lists/%.json: target/release/zigen
|
|
20
|
+
# Funny enough make's basename doesn't strip away the directory...
|
|
21
|
+
./target/release/zigen generate_$(shell basename $@ .json) $@
|
|
22
|
+
|
|
23
|
+
clean:
|
|
24
|
+
cd lists && git clean -f -x -d
|
|
25
|
+
|
|
26
|
+
release:
|
|
27
|
+
cargo build --lib --release
|
|
28
|
+
|
|
29
|
+
test:
|
|
30
|
+
cd ad_hoc_tests/ && make test
|
|
31
|
+
|
|
32
|
+
expensive_tests:
|
|
33
|
+
cd ad_hoc_tests/ && make expensive_tests
|
zilib-0.1.9/NOTES.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Notes
|
|
2
|
+
|
|
3
|
+
- There's apparently no standard way to include data files as part of a
|
|
4
|
+
library, see eg. https://github.com/rust-lang/rfcs/pull/2376 . include_str!
|
|
5
|
+
and include_bytes! isn't efficient. We might have to let the user deal with
|
|
6
|
+
the logistics of interpretting the unihan database. We can provide some
|
|
7
|
+
tooling though.
|
zilib-0.1.9/PKG-INFO
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: zilib
|
|
3
|
+
Version: 0.1.9
|
|
4
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
5
|
+
Classifier: Development Status :: 3 - Alpha
|
|
6
|
+
Classifier: Intended Audience :: Developers
|
|
7
|
+
Classifier: Programming Language :: Python
|
|
8
|
+
Classifier: Programming Language :: Rust
|
|
9
|
+
Classifier: Operating System :: POSIX
|
|
10
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
11
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
14
|
+
|
|
15
|
+
# zilib-python
|
|
16
|
+
|
|
17
|
+
Python wrappers for the Rust zilib library
|
|
18
|
+
|
zilib-0.1.9/README.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# zilib
|
|
2
|
+
|
|
3
|
+
A library written in rust with functions for Han character (and some English
|
|
4
|
+
word) processing.
|
|
5
|
+
|
|
6
|
+
(Not to be confused with zlib, the compression library)
|
|
7
|
+
|
|
8
|
+
## Scope
|
|
9
|
+
|
|
10
|
+
This library contains character-level or word-level language functions used by
|
|
11
|
+
words.hk. We may expand the scope later. Documentation and examples will come
|
|
12
|
+
soon.
|
|
13
|
+
|
|
14
|
+
Originally these functions were implemented in Python in the words.hk codebase,
|
|
15
|
+
but we have ported them to Rust, with optional python bindings (in another
|
|
16
|
+
zilib-python repository (coming soon)).
|
|
17
|
+
|
|
18
|
+
As of writing 2024-02, this repository is under active development. Consider
|
|
19
|
+
the state "unstable" until this notice is removed.
|
|
20
|
+
|
|
21
|
+
Pull requests are welcome as long as the changes are released under the same
|
|
22
|
+
terms as in LICENSE.md. However, from experience (with email enquiries), we may
|
|
23
|
+
sometimes take a while to respond. (Please do not interpret it as disinterest,
|
|
24
|
+
we may simply have forgotten to check our emails...)
|
|
25
|
+
|
|
26
|
+
If you plan to open pull requests to make changes to data files in `list/`,
|
|
27
|
+
please read the section Principles and Philosophies below. Also note that those
|
|
28
|
+
files are under different copyrights/licenses.
|
|
29
|
+
|
|
30
|
+
## License
|
|
31
|
+
|
|
32
|
+
See [LICENSE.md](LICENSE.md)
|
|
33
|
+
|
|
34
|
+
## Principles and Philosophies
|
|
35
|
+
|
|
36
|
+
### Decision Problem Avoidance
|
|
37
|
+
|
|
38
|
+
In our experience, we have found that trying to classify things into clear-cut
|
|
39
|
+
categories often leads to more problems than it solves. This is explained in
|
|
40
|
+
detail by Lau (2019):
|
|
41
|
+
|
|
42
|
+
> There are several classes of problems that have been classified as 'decision
|
|
43
|
+
> problems' by our team. A decision problem is something that requires a
|
|
44
|
+
> judgement, a clear-cut A or B answer. These problems are often not helpful
|
|
45
|
+
> when compiling a dictionary. For example, drawing a line between vegetables
|
|
46
|
+
> and fruits is a decision problem. Some are fairly easy, e.g. orange is
|
|
47
|
+
> definitely a fruit, lettuce is definitely a vegetable; some are less trivial,
|
|
48
|
+
> e.g. tomato. These decision problems do not really give much information to
|
|
49
|
+
> dictionary users. In the latter case, it is preferable for a dictionary to
|
|
50
|
+
> explain the cultural and botanical classifications of tomato, instead of
|
|
51
|
+
> forcing them into inflexible categories.
|
|
52
|
+
>
|
|
53
|
+
> *Lau, C.-M. (2019). Building Cantonese dictionaries using crowdsourcing
|
|
54
|
+
> strategies: The words.hk project. In A. W.-B. Tso (Ed.), Digital humanities and
|
|
55
|
+
> new ways of teaching (pp. 89-107). Singapore: Springer.*
|
|
56
|
+
|
|
57
|
+
Note that in zilib, we work on lists where characters/words are either *in* the
|
|
58
|
+
list or *not*. These are "decision problems" that are unavoidable due to the
|
|
59
|
+
nature of the library. We do not have the luxury of further explaining the
|
|
60
|
+
*apparently* clear-cut decisions (at least not within the API), therefore such
|
|
61
|
+
decisions may look somewhat arbitrary at the edges. This is an unfortunate fact
|
|
62
|
+
when we are building a library that is used by many people who may be each
|
|
63
|
+
operating under slightly different circumstances, contexts and requirements.
|
|
64
|
+
|
|
65
|
+
Nuances and edge cases can often by handled by further complicating the
|
|
66
|
+
"interface" (i.e. the library API), but this makes the library harder to use.
|
|
67
|
+
It is a delicate exercise to balance between a simple and easy-to-use library
|
|
68
|
+
and a library that can handle all edge cases, since the latter will force the
|
|
69
|
+
users of the API to battle with the inherent complexities of language
|
|
70
|
+
processing, in particular the politics and "fractal-esque" aspects of languages
|
|
71
|
+
and dialects.
|
|
72
|
+
|
|
73
|
+
As such, while we try to maintain quality of the reference lists, we decline to
|
|
74
|
+
claim to be authoritative on the decisions made, i.e. we are not here to set
|
|
75
|
+
standards. As a practical note, any changes to our version (i.e. "fork") of the
|
|
76
|
+
library needs to be usable by our own applications (namely words.hk), and we
|
|
77
|
+
may not be able to accommodate all requests. For the principles/philosophies of
|
|
78
|
+
words.hk, see https://words.hk/base/about/ . Given the generally open nature of
|
|
79
|
+
this project (zilib), we encourage users to fork the library and make their own
|
|
80
|
+
changes if they have requirements that are incompatible with ours.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
When testing using ad hoc scripts is easier than testing with unit tests.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
#!/bin/bash
|
|
2
|
+
|
|
3
|
+
GREP=rg
|
|
4
|
+
BSEARCH=../target/debug/binary_search_file
|
|
5
|
+
DATA_FILE=../lists/Unihan_Readings.txt
|
|
6
|
+
tmpfile=$(mktemp)
|
|
7
|
+
|
|
8
|
+
LC_ALL=C sort $DATA_FILE > "$tmpfile"
|
|
9
|
+
echo $tmpfile
|
|
10
|
+
|
|
11
|
+
for ((i=13310;i<=204884+2;i++)); do
|
|
12
|
+
i_hex=$(printf "%04X" $i)
|
|
13
|
+
echo "Testing $i_hex..."
|
|
14
|
+
diff <($BSEARCH $tmpfile "U+$i_hex" ' ' ) <($GREP "^U\\+$i_hex " $tmpfile)
|
|
15
|
+
if [ $? -ne 0 ]; then
|
|
16
|
+
echo "Test failed for $i_hex"
|
|
17
|
+
exit 1
|
|
18
|
+
fi
|
|
19
|
+
|
|
20
|
+
done
|