zilib 0.1.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. zilib-0.1.9/.gitignore +12 -0
  2. zilib-0.1.9/.lastmake +0 -0
  3. zilib-0.1.9/Cargo.lock +232 -0
  4. zilib-0.1.9/Cargo.toml +18 -0
  5. zilib-0.1.9/LICENSE.md +18 -0
  6. zilib-0.1.9/Makefile +33 -0
  7. zilib-0.1.9/NOTES.md +7 -0
  8. zilib-0.1.9/PKG-INFO +18 -0
  9. zilib-0.1.9/README.md +80 -0
  10. zilib-0.1.9/ad_hoc_tests/Makefile +4 -0
  11. zilib-0.1.9/ad_hoc_tests/README.md +1 -0
  12. zilib-0.1.9/ad_hoc_tests/test_binary_search.sh +20 -0
  13. zilib-0.1.9/ad_hoc_tests/test_binary_search_fast.sh +525 -0
  14. zilib-0.1.9/lists/CJKRadicals.txt +283 -0
  15. zilib-0.1.9/lists/HKVariants.txt +63 -0
  16. zilib-0.1.9/lists/TWVariants.txt +39 -0
  17. zilib-0.1.9/lists/charlist.json +1 -0
  18. zilib-0.1.9/lists/edb_charlist.txt +558 -0
  19. zilib-0.1.9/lists/en_unigram_freq.csv.bz2 +0 -0
  20. zilib-0.1.9/lists/english_variants.json +2792 -0
  21. zilib-0.1.9/lists/hfhchan-kVariants.txt +10629 -0
  22. zilib-0.1.9/lists/index.md +242 -0
  23. zilib-0.1.9/lists/varcon.txt.bz2 +0 -0
  24. zilib-0.1.9/lists/wordshk_autoconvert.json +46 -0
  25. zilib-0.1.9/lists/wordshk_autoconvert.txt +100 -0
  26. zilib-0.1.9/lists/wordshk_charlist.txt +179 -0
  27. zilib-0.1.9/lists/wordshk_charset.json +213 -0
  28. zilib-0.1.9/lists/wordshk_variantmap.json +206 -0
  29. zilib-0.1.9/lists/wordshk_variantmap.txt +150 -0
  30. zilib-0.1.9/lists/wordslist.csv +59871 -0
  31. zilib-0.1.9/pyproject.toml +24 -0
  32. zilib-0.1.9/python/zilib/__init__.py +56 -0
  33. zilib-0.1.9/python/zilib/lists/CJKRadicals.txt +283 -0
  34. zilib-0.1.9/python/zilib/lists/HKVariants.txt +63 -0
  35. zilib-0.1.9/python/zilib/lists/TWVariants.txt +39 -0
  36. zilib-0.1.9/python/zilib/lists/charlist.json +1 -0
  37. zilib-0.1.9/python/zilib/lists/edb_charlist.txt +558 -0
  38. zilib-0.1.9/python/zilib/lists/english_variants.json +2792 -0
  39. zilib-0.1.9/python/zilib/lists/hfhchan-kVariants.txt +10629 -0
  40. zilib-0.1.9/python/zilib/lists/index.md +242 -0
  41. zilib-0.1.9/python/zilib/lists/wordshk_autoconvert.json +46 -0
  42. zilib-0.1.9/python/zilib/lists/wordshk_autoconvert.txt +100 -0
  43. zilib-0.1.9/python/zilib/lists/wordshk_charlist.txt +179 -0
  44. zilib-0.1.9/python/zilib/lists/wordshk_charset.json +213 -0
  45. zilib-0.1.9/python/zilib/lists/wordshk_variantmap.json +206 -0
  46. zilib-0.1.9/python/zilib/lists/wordshk_variantmap.txt +150 -0
  47. zilib-0.1.9/python/zilib/lists/wordslist.csv +59871 -0
  48. zilib-0.1.9/scripts/bump.py +47 -0
  49. zilib-0.1.9/src/bin/binary_search_file.rs +71 -0
  50. zilib-0.1.9/src/bin/guess_jyutping.rs +15 -0
  51. zilib-0.1.9/src/bin/radical_sort.rs +54 -0
  52. zilib-0.1.9/src/bin/zicount.rs +47 -0
  53. zilib-0.1.9/src/bin/zigen.rs +443 -0
  54. zilib-0.1.9/src/cantonese.rs +73 -0
  55. zilib-0.1.9/src/cjk.rs +111 -0
  56. zilib-0.1.9/src/common.rs +247 -0
  57. zilib-0.1.9/src/data.rs +361 -0
  58. zilib-0.1.9/src/english.rs +456 -0
  59. zilib-0.1.9/src/lib.rs +8 -0
  60. zilib-0.1.9/src/ruby_match.rs +371 -0
  61. zilib-0.1.9/src/segmentation.rs +158 -0
  62. zilib-0.1.9/zilib-python/.cargo/config +12 -0
  63. zilib-0.1.9/zilib-python/Cargo.lock +463 -0
  64. zilib-0.1.9/zilib-python/Cargo.toml +16 -0
  65. zilib-0.1.9/zilib-python/Makefile +11 -0
  66. zilib-0.1.9/zilib-python/README.md +3 -0
  67. zilib-0.1.9/zilib-python/copy_function_definitions.py +106 -0
  68. zilib-0.1.9/zilib-python/copy_packaged_lists +36 -0
  69. zilib-0.1.9/zilib-python/python/zilib/__init__.py +56 -0
  70. zilib-0.1.9/zilib-python/python/zilib/lists/CJKRadicals.txt +283 -0
  71. zilib-0.1.9/zilib-python/python/zilib/lists/HKVariants.txt +63 -0
  72. zilib-0.1.9/zilib-python/python/zilib/lists/TWVariants.txt +39 -0
  73. zilib-0.1.9/zilib-python/python/zilib/lists/charlist.json +1 -0
  74. zilib-0.1.9/zilib-python/python/zilib/lists/edb_charlist.txt +558 -0
  75. zilib-0.1.9/zilib-python/python/zilib/lists/english_variants.json +2792 -0
  76. zilib-0.1.9/zilib-python/python/zilib/lists/hfhchan-kVariants.txt +10629 -0
  77. zilib-0.1.9/zilib-python/python/zilib/lists/index.md +242 -0
  78. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_autoconvert.json +46 -0
  79. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_autoconvert.txt +100 -0
  80. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_charlist.txt +179 -0
  81. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_charset.json +213 -0
  82. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_variantmap.json +206 -0
  83. zilib-0.1.9/zilib-python/python/zilib/lists/wordshk_variantmap.txt +150 -0
  84. zilib-0.1.9/zilib-python/python/zilib/lists/wordslist.csv +59871 -0
  85. zilib-0.1.9/zilib-python/src/lib.rs +189 -0
  86. zilib-0.1.9/zilib-python/tests/test_general.py +63 -0
  87. zilib-0.1.9/zilib-python/tests/test_wordshk_original.py +423 -0
zilib-0.1.9/.gitignore ADDED
@@ -0,0 +1,12 @@
1
+ /target
2
+ /python/target
3
+ /venv
4
+ /lists/Unihan.zip
5
+ /lists/Unihan_DictionaryIndices.txt
6
+ /lists/Unihan_DictionaryLikeData.txt
7
+ /lists/Unihan_IRGSources.txt
8
+ /lists/Unihan_NumericValues.txt
9
+ /lists/Unihan_OtherMappings.txt
10
+ /lists/Unihan_RadicalStrokeCounts.txt
11
+ /lists/Unihan_Readings.txt
12
+ /lists/Unihan_Variants.txt
zilib-0.1.9/.lastmake ADDED
File without changes
zilib-0.1.9/Cargo.lock ADDED
@@ -0,0 +1,232 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 3
4
+
5
+ [[package]]
6
+ name = "aho-corasick"
7
+ version = "1.1.2"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
10
+ dependencies = [
11
+ "memchr",
12
+ ]
13
+
14
+ [[package]]
15
+ name = "bzip2"
16
+ version = "0.4.4"
17
+ source = "registry+https://github.com/rust-lang/crates.io-index"
18
+ checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
19
+ dependencies = [
20
+ "bzip2-sys",
21
+ "libc",
22
+ ]
23
+
24
+ [[package]]
25
+ name = "bzip2-sys"
26
+ version = "0.1.11+1.0.8"
27
+ source = "registry+https://github.com/rust-lang/crates.io-index"
28
+ checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
29
+ dependencies = [
30
+ "cc",
31
+ "libc",
32
+ "pkg-config",
33
+ ]
34
+
35
+ [[package]]
36
+ name = "cc"
37
+ version = "1.0.83"
38
+ source = "registry+https://github.com/rust-lang/crates.io-index"
39
+ checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
40
+ dependencies = [
41
+ "libc",
42
+ ]
43
+
44
+ [[package]]
45
+ name = "cfg-if"
46
+ version = "1.0.0"
47
+ source = "registry+https://github.com/rust-lang/crates.io-index"
48
+ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
49
+
50
+ [[package]]
51
+ name = "csv"
52
+ version = "1.3.0"
53
+ source = "registry+https://github.com/rust-lang/crates.io-index"
54
+ checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
55
+ dependencies = [
56
+ "csv-core",
57
+ "itoa",
58
+ "ryu",
59
+ "serde",
60
+ ]
61
+
62
+ [[package]]
63
+ name = "csv-core"
64
+ version = "0.1.11"
65
+ source = "registry+https://github.com/rust-lang/crates.io-index"
66
+ checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
67
+ dependencies = [
68
+ "memchr",
69
+ ]
70
+
71
+ [[package]]
72
+ name = "encoding_rs"
73
+ version = "0.8.33"
74
+ source = "registry+https://github.com/rust-lang/crates.io-index"
75
+ checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
76
+ dependencies = [
77
+ "cfg-if",
78
+ ]
79
+
80
+ [[package]]
81
+ name = "encoding_rs_io"
82
+ version = "0.1.7"
83
+ source = "registry+https://github.com/rust-lang/crates.io-index"
84
+ checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
85
+ dependencies = [
86
+ "encoding_rs",
87
+ ]
88
+
89
+ [[package]]
90
+ name = "itoa"
91
+ version = "1.0.10"
92
+ source = "registry+https://github.com/rust-lang/crates.io-index"
93
+ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
94
+
95
+ [[package]]
96
+ name = "libc"
97
+ version = "0.2.153"
98
+ source = "registry+https://github.com/rust-lang/crates.io-index"
99
+ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
100
+
101
+ [[package]]
102
+ name = "memchr"
103
+ version = "2.7.1"
104
+ source = "registry+https://github.com/rust-lang/crates.io-index"
105
+ checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
106
+
107
+ [[package]]
108
+ name = "pkg-config"
109
+ version = "0.3.29"
110
+ source = "registry+https://github.com/rust-lang/crates.io-index"
111
+ checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb"
112
+
113
+ [[package]]
114
+ name = "proc-macro2"
115
+ version = "1.0.78"
116
+ source = "registry+https://github.com/rust-lang/crates.io-index"
117
+ checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
118
+ dependencies = [
119
+ "unicode-ident",
120
+ ]
121
+
122
+ [[package]]
123
+ name = "quote"
124
+ version = "1.0.35"
125
+ source = "registry+https://github.com/rust-lang/crates.io-index"
126
+ checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
127
+ dependencies = [
128
+ "proc-macro2",
129
+ ]
130
+
131
+ [[package]]
132
+ name = "regex"
133
+ version = "1.10.3"
134
+ source = "registry+https://github.com/rust-lang/crates.io-index"
135
+ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
136
+ dependencies = [
137
+ "aho-corasick",
138
+ "memchr",
139
+ "regex-automata",
140
+ "regex-syntax",
141
+ ]
142
+
143
+ [[package]]
144
+ name = "regex-automata"
145
+ version = "0.4.5"
146
+ source = "registry+https://github.com/rust-lang/crates.io-index"
147
+ checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
148
+ dependencies = [
149
+ "aho-corasick",
150
+ "memchr",
151
+ "regex-syntax",
152
+ ]
153
+
154
+ [[package]]
155
+ name = "regex-syntax"
156
+ version = "0.8.2"
157
+ source = "registry+https://github.com/rust-lang/crates.io-index"
158
+ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
159
+
160
+ [[package]]
161
+ name = "ryu"
162
+ version = "1.0.16"
163
+ source = "registry+https://github.com/rust-lang/crates.io-index"
164
+ checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
165
+
166
+ [[package]]
167
+ name = "serde"
168
+ version = "1.0.196"
169
+ source = "registry+https://github.com/rust-lang/crates.io-index"
170
+ checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
171
+ dependencies = [
172
+ "serde_derive",
173
+ ]
174
+
175
+ [[package]]
176
+ name = "serde_derive"
177
+ version = "1.0.196"
178
+ source = "registry+https://github.com/rust-lang/crates.io-index"
179
+ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
180
+ dependencies = [
181
+ "proc-macro2",
182
+ "quote",
183
+ "syn",
184
+ ]
185
+
186
+ [[package]]
187
+ name = "serde_json"
188
+ version = "1.0.113"
189
+ source = "registry+https://github.com/rust-lang/crates.io-index"
190
+ checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
191
+ dependencies = [
192
+ "itoa",
193
+ "ryu",
194
+ "serde",
195
+ ]
196
+
197
+ [[package]]
198
+ name = "syn"
199
+ version = "2.0.48"
200
+ source = "registry+https://github.com/rust-lang/crates.io-index"
201
+ checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
202
+ dependencies = [
203
+ "proc-macro2",
204
+ "quote",
205
+ "unicode-ident",
206
+ ]
207
+
208
+ [[package]]
209
+ name = "unicode-ident"
210
+ version = "1.0.12"
211
+ source = "registry+https://github.com/rust-lang/crates.io-index"
212
+ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
213
+
214
+ [[package]]
215
+ name = "unicode_categories"
216
+ version = "0.1.1"
217
+ source = "registry+https://github.com/rust-lang/crates.io-index"
218
+ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
219
+
220
+ [[package]]
221
+ name = "zilib"
222
+ version = "0.1.9"
223
+ dependencies = [
224
+ "bzip2",
225
+ "csv",
226
+ "encoding_rs",
227
+ "encoding_rs_io",
228
+ "regex",
229
+ "serde",
230
+ "serde_json",
231
+ "unicode_categories",
232
+ ]
zilib-0.1.9/Cargo.toml ADDED
@@ -0,0 +1,18 @@
1
+ [package]
2
+ name = "zilib"
3
+ version = "0.1.9"
4
+ edition = "2021"
5
+
6
+ [dependencies]
7
+ bzip2 = "0.4.4"
8
+ csv = "1.3.0"
9
+ encoding_rs = "0.8.33"
10
+ encoding_rs_io = "0.1.7"
11
+ regex = "1.10.3"
12
+ serde = { version = "1.0.196", features = ["derive"] }
13
+ serde_json = "1.0.113"
14
+ unicode_categories = "0.1.1"
15
+
16
+ [features]
17
+ default = []
18
+ downloaded_data = []
zilib-0.1.9/LICENSE.md ADDED
@@ -0,0 +1,18 @@
1
+ ## Overview
2
+
3
+ - Code in this repository is released under the following license unless otherwise stated.
4
+
5
+ - For information on data files, see [lists/index.md](lists/index.md) instead.
6
+
7
+
8
+ ## License
9
+
10
+ ```
11
+ Copyright 2024 Hong Kong Lexicography Limited.
12
+
13
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
14
+
15
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
18
+ ```
zilib-0.1.9/Makefile ADDED
@@ -0,0 +1,33 @@
1
+ all: zigen_data rust_all
2
+ rust_all: zigen_data lists/wordslist.csv
3
+ cargo build --lib
4
+ lint:
5
+ cargo clippy
6
+
7
+ target/release/%: src/*.rs src/bin/*.rs lists/wordslist.csv
8
+ cargo build --release --bins --no-default-features --features downloaded_data
9
+
10
+ zigen_data: lists/Unihan.zip lists/CJKRadicals.txt lists/english_variants.json lists/wordshk_charset.json lists/wordshk_variantmap.json lists/wordshk_autoconvert.json
11
+
12
+ lists/wordslist.csv:
13
+ cd lists && curl -O https://words.hk/faiman/analysis/wordslist.csv
14
+ lists/Unihan.zip:
15
+ cd lists && curl -O https://www.unicode.org/Public/UCD/latest/ucd/Unihan.zip && unzip Unihan.zip
16
+ lists/CJKRadicals.txt:
17
+ cd lists && curl -O https://www.unicode.org/Public/UCD/latest/ucd/CJKRadicals.txt
18
+
19
+ lists/%.json: target/release/zigen
20
+ # Funny enough make's basename doesn't strip away the directory...
21
+ ./target/release/zigen generate_$(shell basename $@ .json) $@
22
+
23
+ clean:
24
+ cd lists && git clean -f -x -d
25
+
26
+ release:
27
+ cargo build --lib --release
28
+
29
+ test:
30
+ cd ad_hoc_tests/ && make test
31
+
32
+ expensive_tests:
33
+ cd ad_hoc_tests/ && make expensive_tests
zilib-0.1.9/NOTES.md ADDED
@@ -0,0 +1,7 @@
1
+ # Notes
2
+
3
+ - There's apparently no standard way to include data files as part of a
4
+ library, see eg. https://github.com/rust-lang/rfcs/pull/2376 . include_str!
5
+ and include_bytes! isn't efficient. We might have to let the user deal with
6
+ the logistics of interpretting the unihan database. We can provide some
7
+ tooling though.
zilib-0.1.9/PKG-INFO ADDED
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.1
2
+ Name: zilib
3
+ Version: 0.1.9
4
+ Classifier: License :: OSI Approved :: MIT License
5
+ Classifier: Development Status :: 3 - Alpha
6
+ Classifier: Intended Audience :: Developers
7
+ Classifier: Programming Language :: Python
8
+ Classifier: Programming Language :: Rust
9
+ Classifier: Operating System :: POSIX
10
+ Classifier: Operating System :: MacOS :: MacOS X
11
+ Requires-Dist: pytest ; extra == 'dev'
12
+ Provides-Extra: dev
13
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
14
+
15
+ # zilib-python
16
+
17
+ Python wrappers for the Rust zilib library
18
+
zilib-0.1.9/README.md ADDED
@@ -0,0 +1,80 @@
1
+ # zilib
2
+
3
+ A library written in rust with functions for Han character (and some English
4
+ word) processing.
5
+
6
+ (Not to be confused with zlib, the compression library)
7
+
8
+ ## Scope
9
+
10
+ This library contains character-level or word-level language functions used by
11
+ words.hk. We may expand the scope later. Documentation and examples will come
12
+ soon.
13
+
14
+ Originally these functions were implemented in Python in the words.hk codebase,
15
+ but we have ported them to Rust, with optional python bindings (in another
16
+ zilib-python repository (coming soon)).
17
+
18
+ As of writing 2024-02, this repository is under active development. Consider
19
+ the state "unstable" until this notice is removed.
20
+
21
+ Pull requests are welcome as long as the changes are released under the same
22
+ terms as in LICENSE.md. However, from experience (with email enquiries), we may
23
+ sometimes take a while to respond. (Please do not interpret it as disinterest,
24
+ we may simply have forgotten to check our emails...)
25
+
26
+ If you plan to open pull requests to make changes to data files in `list/`,
27
+ please read the section Principles and Philosophies below. Also note that those
28
+ files are under different copyrights/licenses.
29
+
30
+ ## License
31
+
32
+ See [LICENSE.md](LICENSE.md)
33
+
34
+ ## Principles and Philosophies
35
+
36
+ ### Decision Problem Avoidance
37
+
38
+ In our experience, we have found that trying to classify things into clear-cut
39
+ categories often leads to more problems than it solves. This is explained in
40
+ detail by Lau (2019):
41
+
42
+ > There are several classes of problems that have been classified as 'decision
43
+ > problems' by our team. A decision problem is something that requires a
44
+ > judgement, a clear-cut A or B answer. These problems are often not helpful
45
+ > when compiling a dictionary. For example, drawing a line between vegetables
46
+ > and fruits is a decision problem. Some are fairly easy, e.g. orange is
47
+ > definitely a fruit, lettuce is definitely a vegetable; some are less trivial,
48
+ > e.g. tomato. These decision problems do not really give much information to
49
+ > dictionary users. In the latter case, it is preferable for a dictionary to
50
+ > explain the cultural and botanical classifications of tomato, instead of
51
+ > forcing them into inflexible categories.
52
+ >
53
+ > *Lau, C.-M. (2019). Building Cantonese dictionaries using crowdsourcing
54
+ > strategies: The words.hk project. In A. W.-B. Tso (Ed.), Digital humanities and
55
+ > new ways of teaching (pp. 89-107). Singapore: Springer.*
56
+
57
+ Note that in zilib, we work on lists where characters/words are either *in* the
58
+ list or *not*. These are "decision problems" that are unavoidable due to the
59
+ nature of the library. We do not have the luxury of further explaining the
60
+ *apparently* clear-cut decisions (at least not within the API), therefore such
61
+ decisions may look somewhat arbitrary at the edges. This is an unfortunate fact
62
+ when we are building a library that is used by many people who may be each
63
+ operating under slightly different circumstances, contexts and requirements.
64
+
65
+ Nuances and edge cases can often by handled by further complicating the
66
+ "interface" (i.e. the library API), but this makes the library harder to use.
67
+ It is a delicate exercise to balance between a simple and easy-to-use library
68
+ and a library that can handle all edge cases, since the latter will force the
69
+ users of the API to battle with the inherent complexities of language
70
+ processing, in particular the politics and "fractal-esque" aspects of languages
71
+ and dialects.
72
+
73
+ As such, while we try to maintain quality of the reference lists, we decline to
74
+ claim to be authoritative on the decisions made, i.e. we are not here to set
75
+ standards. As a practical note, any changes to our version (i.e. "fork") of the
76
+ library needs to be usable by our own applications (namely words.hk), and we
77
+ may not be able to accommodate all requests. For the principles/philosophies of
78
+ words.hk, see https://words.hk/base/about/ . Given the generally open nature of
79
+ this project (zilib), we encourage users to fork the library and make their own
80
+ changes if they have requirements that are incompatible with ours.
@@ -0,0 +1,4 @@
1
+ test:
2
+ bash test_binary_search_fast.sh
3
+ expensive_test: test
4
+ bash test_binary_search.sh
@@ -0,0 +1 @@
1
+ When testing using ad hoc scripts is easier than testing with unit tests.
@@ -0,0 +1,20 @@
1
+ #!/bin/bash
2
+
3
+ GREP=rg
4
+ BSEARCH=../target/debug/binary_search_file
5
+ DATA_FILE=../lists/Unihan_Readings.txt
6
+ tmpfile=$(mktemp)
7
+
8
+ LC_ALL=C sort $DATA_FILE > "$tmpfile"
9
+ echo $tmpfile
10
+
11
+ for ((i=13310;i<=204884+2;i++)); do
12
+ i_hex=$(printf "%04X" $i)
13
+ echo "Testing $i_hex..."
14
+ diff <($BSEARCH $tmpfile "U+$i_hex" ' ' ) <($GREP "^U\\+$i_hex " $tmpfile)
15
+ if [ $? -ne 0 ]; then
16
+ echo "Test failed for $i_hex"
17
+ exit 1
18
+ fi
19
+
20
+ done