rustling 0.7.0__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. {rustling-0.7.0 → rustling-0.8.0}/.gitignore +3 -0
  2. {rustling-0.7.0 → rustling-0.8.0}/CHANGELOG.md +13 -0
  3. {rustling-0.7.0 → rustling-0.8.0}/Cargo.lock +16 -19
  4. {rustling-0.7.0 → rustling-0.8.0}/Cargo.toml +2 -1
  5. {rustling-0.7.0 → rustling-0.8.0}/PKG-INFO +22 -26
  6. {rustling-0.7.0 → rustling-0.8.0}/README.md +21 -25
  7. rustling-0.8.0/benchmarks/README.md +132 -0
  8. rustling-0.8.0/benchmarks/run_conllu.py +300 -0
  9. rustling-0.8.0/benchmarks/run_elan.py +299 -0
  10. rustling-0.8.0/benchmarks/run_textgrid.py +320 -0
  11. rustling-0.7.0/benchmarks/update_docs.py → rustling-0.8.0/benchmarks/update_readme.py +86 -66
  12. {rustling-0.7.0 → rustling-0.8.0}/pyproject.toml +2 -0
  13. rustling-0.8.0/python/docs/_static/brown_eve_mlum.png +0 -0
  14. {rustling-0.7.0 → rustling-0.8.0}/python/docs/api.rst +4 -0
  15. rustling-0.8.0/python/docs/chat/formats.rst +219 -0
  16. rustling-0.8.0/python/docs/chat/frequencies.rst +99 -0
  17. rustling-0.8.0/python/docs/chat/headers.rst +171 -0
  18. rustling-0.8.0/python/docs/chat/measures.rst +62 -0
  19. rustling-0.8.0/python/docs/chat/quickstart.rst +349 -0
  20. rustling-0.8.0/python/docs/chat/read.rst +479 -0
  21. rustling-0.8.0/python/docs/chat/transcriptions.rst +626 -0
  22. rustling-0.8.0/python/docs/chat/write.rst +48 -0
  23. rustling-0.8.0/python/docs/chat.rst +19 -0
  24. rustling-0.8.0/python/docs/conllu.rst +221 -0
  25. rustling-0.8.0/python/docs/elan.rst +275 -0
  26. rustling-0.8.0/python/docs/index.rst +104 -0
  27. rustling-0.8.0/python/docs/srt.rst +240 -0
  28. rustling-0.8.0/python/docs/textgrid.rst +288 -0
  29. rustling-0.8.0/python/rustling/__init__.py +36 -0
  30. rustling-0.8.0/python/rustling/_lib_name.pyi +3 -0
  31. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/chat/__init__.py +2 -0
  32. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/chat/__init__.pyi +316 -11
  33. rustling-0.8.0/python/rustling/chat/_read_chat.py +78 -0
  34. rustling-0.8.0/python/rustling/conllu/__init__.py +13 -0
  35. rustling-0.8.0/python/rustling/conllu/__init__.pyi +326 -0
  36. rustling-0.8.0/python/rustling/conllu/_read_conllu.py +53 -0
  37. rustling-0.8.0/python/rustling/elan/__init__.py +19 -0
  38. rustling-0.8.0/python/rustling/elan/__init__.pyi +442 -0
  39. rustling-0.8.0/python/rustling/elan/_read_elan.py +53 -0
  40. rustling-0.8.0/python/rustling/srt/__init__.py +11 -0
  41. rustling-0.8.0/python/rustling/srt/__init__.pyi +345 -0
  42. rustling-0.8.0/python/rustling/srt/_read_srt.py +53 -0
  43. rustling-0.8.0/python/rustling/textgrid/__init__.py +23 -0
  44. rustling-0.8.0/python/rustling/textgrid/__init__.pyi +348 -0
  45. rustling-0.8.0/python/rustling/textgrid/_read_textgrid.py +54 -0
  46. {rustling-0.7.0 → rustling-0.8.0}/python/stubtest_allowlist.txt +31 -0
  47. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_chat/test_chat.py +153 -26
  48. rustling-0.8.0/python/tests/test_conllu/conftest.py +45 -0
  49. rustling-0.8.0/python/tests/test_conllu/test_conllu.py +276 -0
  50. rustling-0.8.0/python/tests/test_elan/conftest.py +33 -0
  51. rustling-0.8.0/python/tests/test_elan/test_elan.py +546 -0
  52. rustling-0.8.0/python/tests/test_textgrid/conftest.py +0 -0
  53. rustling-0.8.0/python/tests/test_textgrid/test_textgrid.py +398 -0
  54. rustling-0.8.0/src/chat/conllu_writer.rs +147 -0
  55. rustling-0.8.0/src/chat/elan_writer.rs +486 -0
  56. {rustling-0.7.0 → rustling-0.8.0}/src/chat/mod.rs +4 -0
  57. {rustling-0.7.0 → rustling-0.8.0}/src/chat/reader.rs +380 -39
  58. {rustling-0.7.0 → rustling-0.8.0}/src/chat/reader_py.rs +239 -9
  59. rustling-0.8.0/src/chat/srt_writer.rs +324 -0
  60. rustling-0.8.0/src/chat/textgrid_writer.rs +201 -0
  61. {rustling-0.7.0 → rustling-0.8.0}/src/chat/validation.rs +35 -5
  62. rustling-0.8.0/src/conllu/chat_writer.rs +222 -0
  63. rustling-0.8.0/src/conllu/mod.rs +885 -0
  64. rustling-0.8.0/src/conllu/reader_py.rs +547 -0
  65. rustling-0.8.0/src/elan/chat_writer.rs +414 -0
  66. rustling-0.8.0/src/elan/mod.rs +32 -0
  67. rustling-0.8.0/src/elan/reader.rs +1182 -0
  68. rustling-0.8.0/src/elan/reader_py.rs +629 -0
  69. rustling-0.8.0/src/elan/srt_writer.rs +242 -0
  70. rustling-0.8.0/src/elan/textgrid_writer.rs +155 -0
  71. {rustling-0.7.0 → rustling-0.8.0}/src/lib.rs +10 -0
  72. rustling-0.8.0/src/sources/cache.rs +76 -0
  73. rustling-0.8.0/src/sources/git.rs +123 -0
  74. rustling-0.8.0/src/sources/mod.rs +57 -0
  75. rustling-0.8.0/src/sources/url.rs +68 -0
  76. rustling-0.8.0/src/srt/chat_writer.rs +140 -0
  77. rustling-0.8.0/src/srt/elan_writer.rs +211 -0
  78. rustling-0.8.0/src/srt/mod.rs +986 -0
  79. rustling-0.8.0/src/srt/reader_py.rs +523 -0
  80. rustling-0.8.0/src/srt/textgrid_writer.rs +122 -0
  81. rustling-0.8.0/src/textgrid/chat_writer.rs +219 -0
  82. rustling-0.8.0/src/textgrid/elan_writer.rs +245 -0
  83. rustling-0.8.0/src/textgrid/mod.rs +34 -0
  84. rustling-0.8.0/src/textgrid/reader.rs +1691 -0
  85. rustling-0.8.0/src/textgrid/reader_py.rs +750 -0
  86. rustling-0.8.0/src/textgrid/srt_writer.rs +208 -0
  87. rustling-0.7.0/benchmarks/README.md +0 -89
  88. rustling-0.7.0/python/docs/chat.rst +0 -7
  89. rustling-0.7.0/python/docs/index.rst +0 -123
  90. rustling-0.7.0/python/rustling/__init__.py +0 -11
  91. {rustling-0.7.0 → rustling-0.8.0}/.flake8 +0 -0
  92. {rustling-0.7.0 → rustling-0.8.0}/.gitattributes +0 -0
  93. {rustling-0.7.0 → rustling-0.8.0}/.github/dependabot.yml +0 -0
  94. {rustling-0.7.0 → rustling-0.8.0}/.github/workflows/build-wheel.yml +0 -0
  95. {rustling-0.7.0 → rustling-0.8.0}/.github/workflows/python.yml +0 -0
  96. {rustling-0.7.0 → rustling-0.8.0}/.github/workflows/release.yml +0 -0
  97. {rustling-0.7.0 → rustling-0.8.0}/.github/workflows/rust.yml +0 -0
  98. {rustling-0.7.0 → rustling-0.8.0}/.pre-commit-config.yaml +0 -0
  99. {rustling-0.7.0 → rustling-0.8.0}/.readthedocs.yaml +0 -0
  100. {rustling-0.7.0 → rustling-0.8.0}/LICENSE.md +0 -0
  101. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/common/__init__.py +0 -0
  102. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/common/data.py +0 -0
  103. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/run_chat.py +0 -0
  104. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/run_hmm.py +0 -0
  105. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/run_lm.py +0 -0
  106. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/run_perceptron_pos_tagger.py +0 -0
  107. {rustling-0.7.0 → rustling-0.8.0}/benchmarks/run_wordseg.py +0 -0
  108. {rustling-0.7.0 → rustling-0.8.0}/build.rs +0 -0
  109. {rustling-0.7.0 → rustling-0.8.0}/python/docs/Makefile +0 -0
  110. {rustling-0.7.0 → rustling-0.8.0}/python/docs/README.md +0 -0
  111. {rustling-0.7.0 → rustling-0.8.0}/python/docs/_static/favicon.ico +0 -0
  112. {rustling-0.7.0 → rustling-0.8.0}/python/docs/_static/logo-with-text.svg +0 -0
  113. {rustling-0.7.0 → rustling-0.8.0}/python/docs/_templates/copyright.html +0 -0
  114. {rustling-0.7.0 → rustling-0.8.0}/python/docs/conf.py +0 -0
  115. {rustling-0.7.0 → rustling-0.8.0}/python/docs/hmm.rst +0 -0
  116. {rustling-0.7.0 → rustling-0.8.0}/python/docs/lm.rst +0 -0
  117. {rustling-0.7.0 → rustling-0.8.0}/python/docs/make.bat +0 -0
  118. {rustling-0.7.0 → rustling-0.8.0}/python/docs/ngram.rst +0 -0
  119. {rustling-0.7.0 → rustling-0.8.0}/python/docs/perceptron_pos_tagger.rst +0 -0
  120. {rustling-0.7.0 → rustling-0.8.0}/python/docs/requirements.txt +0 -0
  121. {rustling-0.7.0 → rustling-0.8.0}/python/docs/wordseg.rst +0 -0
  122. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/hmm/__init__.py +0 -0
  123. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/hmm/__init__.pyi +0 -0
  124. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/lm/__init__.py +0 -0
  125. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/lm/__init__.pyi +0 -0
  126. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/ngram/__init__.py +0 -0
  127. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/ngram/__init__.pyi +0 -0
  128. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/perceptron_pos_tagger/__init__.py +0 -0
  129. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/perceptron_pos_tagger/__init__.pyi +0 -0
  130. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/py.typed +0 -0
  131. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/seq_feature/__init__.py +0 -0
  132. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/seq_feature/__init__.pyi +0 -0
  133. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/wordseg/__init__.py +0 -0
  134. {rustling-0.7.0 → rustling-0.8.0}/python/rustling/wordseg/__init__.pyi +0 -0
  135. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_benchmarks.py +0 -0
  136. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_chat/conftest.py +0 -0
  137. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_hmm.py +0 -0
  138. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_lm.py +0 -0
  139. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_ngram.py +0 -0
  140. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_perceptron_pos_tagger.py +0 -0
  141. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_wordseg/test_dag_hmm_segmenter.py +0 -0
  142. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_wordseg/test_hmm_segmenter.py +0 -0
  143. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_wordseg/test_longest_string_matching.py +0 -0
  144. {rustling-0.7.0 → rustling-0.8.0}/python/tests/test_wordseg/test_random_segmenter.py +0 -0
  145. {rustling-0.7.0 → rustling-0.8.0}/src/chat/clean_utterance.rs +0 -0
  146. {rustling-0.7.0 → rustling-0.8.0}/src/chat/header.rs +0 -0
  147. {rustling-0.7.0 → rustling-0.8.0}/src/chat/header_py.rs +0 -0
  148. {rustling-0.7.0 → rustling-0.8.0}/src/chat/ipsyn.rs +0 -0
  149. {rustling-0.7.0 → rustling-0.8.0}/src/chat/utterance.rs +0 -0
  150. {rustling-0.7.0 → rustling-0.8.0}/src/chat/utterance_py.rs +0 -0
  151. {rustling-0.7.0 → rustling-0.8.0}/src/hmm/mod.rs +0 -0
  152. {rustling-0.7.0 → rustling-0.8.0}/src/hmm/model.fbs +0 -0
  153. {rustling-0.7.0 → rustling-0.8.0}/src/hmm/model_generated.rs +0 -0
  154. {rustling-0.7.0 → rustling-0.8.0}/src/hmm/py.rs +0 -0
  155. {rustling-0.7.0 → rustling-0.8.0}/src/lm/mod.rs +0 -0
  156. {rustling-0.7.0 → rustling-0.8.0}/src/lm/model.fbs +0 -0
  157. {rustling-0.7.0 → rustling-0.8.0}/src/lm/model_generated.rs +0 -0
  158. {rustling-0.7.0 → rustling-0.8.0}/src/lm/py.rs +0 -0
  159. {rustling-0.7.0 → rustling-0.8.0}/src/ngram/mod.rs +0 -0
  160. {rustling-0.7.0 → rustling-0.8.0}/src/ngram/py.rs +0 -0
  161. {rustling-0.7.0 → rustling-0.8.0}/src/perceptron_pos_tagger/mod.rs +0 -0
  162. {rustling-0.7.0 → rustling-0.8.0}/src/perceptron_pos_tagger/model.fbs +0 -0
  163. {rustling-0.7.0 → rustling-0.8.0}/src/perceptron_pos_tagger/model_generated.rs +0 -0
  164. {rustling-0.7.0 → rustling-0.8.0}/src/perceptron_pos_tagger/py.rs +0 -0
  165. {rustling-0.7.0 → rustling-0.8.0}/src/persistence.rs +0 -0
  166. {rustling-0.7.0 → rustling-0.8.0}/src/prelude.rs +0 -0
  167. {rustling-0.7.0 → rustling-0.8.0}/src/seq_feature/mod.rs +0 -0
  168. {rustling-0.7.0 → rustling-0.8.0}/src/seq_feature/py.rs +0 -0
  169. {rustling-0.7.0 → rustling-0.8.0}/src/trie.rs +0 -0
  170. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/bmes.rs +0 -0
  171. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/dag_hmm.rs +0 -0
  172. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/dag_hmm_py.rs +0 -0
  173. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/hmm.rs +0 -0
  174. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/hmm_py.rs +0 -0
  175. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/longest_string_matching/mod.rs +0 -0
  176. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/longest_string_matching/model.fbs +0 -0
  177. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/longest_string_matching/model_generated.rs +0 -0
  178. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/longest_string_matching/py.rs +0 -0
  179. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/mod.rs +0 -0
  180. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/random_segmenter.rs +0 -0
  181. {rustling-0.7.0 → rustling-0.8.0}/src/wordseg/random_segmenter_py.rs +0 -0
@@ -77,3 +77,6 @@ benchmarks/.results/
77
77
  # This is a library, not an application.
78
78
  Cargo.lock
79
79
  uv.lock
80
+
81
+ # From maturin develop
82
+ python/rustling/librustling.dylib.dSYM/
@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.8.0] - 2026-03-20
11
+
12
+ ### Added
13
+
14
+ - Support for data formats:
15
+ * CoNLL-U for Universal Dependencies
16
+ * ELAN for annotated multimedia data
17
+ * TextGrid for Praat annotations
18
+ * SRT for subtitles
19
+ - CHAT data handling:
20
+ * Added a convenience function `read_chat.
21
+ * Added `from_git` and `from_url` methods for remote data sources.
22
+
10
23
  ## [0.7.0] - 2026-03-14
11
24
 
12
25
  ### Added
@@ -380,9 +380,9 @@ dependencies = [
380
380
 
381
381
  [[package]]
382
382
  name = "itoa"
383
- version = "1.0.17"
383
+ version = "1.0.18"
384
384
  source = "registry+https://github.com/rust-lang/crates.io-index"
385
- checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
385
+ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
386
386
 
387
387
  [[package]]
388
388
  name = "jobserver"
@@ -582,6 +582,16 @@ dependencies = [
582
582
  "syn",
583
583
  ]
584
584
 
585
+ [[package]]
586
+ name = "quick-xml"
587
+ version = "0.39.2"
588
+ source = "registry+https://github.com/rust-lang/crates.io-index"
589
+ checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d"
590
+ dependencies = [
591
+ "memchr",
592
+ "serde",
593
+ ]
594
+
585
595
  [[package]]
586
596
  name = "quote"
587
597
  version = "1.0.45"
@@ -699,11 +709,12 @@ dependencies = [
699
709
 
700
710
  [[package]]
701
711
  name = "rustling"
702
- version = "0.7.0"
712
+ version = "0.8.0"
703
713
  dependencies = [
704
714
  "fancy-regex",
705
715
  "flatbuffers",
706
716
  "pyo3",
717
+ "quick-xml",
707
718
  "rand",
708
719
  "rayon",
709
720
  "regex",
@@ -1136,26 +1147,12 @@ name = "zeroize"
1136
1147
  version = "1.8.2"
1137
1148
  source = "registry+https://github.com/rust-lang/crates.io-index"
1138
1149
  checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
1139
- dependencies = [
1140
- "zeroize_derive",
1141
- ]
1142
-
1143
- [[package]]
1144
- name = "zeroize_derive"
1145
- version = "1.4.3"
1146
- source = "registry+https://github.com/rust-lang/crates.io-index"
1147
- checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e"
1148
- dependencies = [
1149
- "proc-macro2",
1150
- "quote",
1151
- "syn",
1152
- ]
1153
1150
 
1154
1151
  [[package]]
1155
1152
  name = "zip"
1156
- version = "8.2.0"
1153
+ version = "8.3.0"
1157
1154
  source = "registry+https://github.com/rust-lang/crates.io-index"
1158
- checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004"
1155
+ checksum = "4a243cfad17427fc077f529da5a95abe4e94fd2bfdb601611870a6557cc67657"
1159
1156
  dependencies = [
1160
1157
  "aes",
1161
1158
  "bzip2",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "rustling"
3
- version = "0.7.0"
3
+ version = "0.8.0"
4
4
  edition = "2024"
5
5
  description = "A blazingly fast library for computational linguistics"
6
6
  license = "MIT"
@@ -33,6 +33,7 @@ serde_json = "1.0"
33
33
  uuid = { version = "1", features = ["v4"] }
34
34
  walkdir = "2"
35
35
  zip = "8"
36
+ quick-xml = { version = "0.39", features = ["serialize"] }
36
37
  zstd = "0.13"
37
38
 
38
39
  [dev-dependencies]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rustling
3
- Version: 0.7.0
3
+ Version: 0.8.0
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Education
@@ -42,41 +42,37 @@ Project-URL: Source, https://github.com/jacksonllee/rustling
42
42
  <br>
43
43
 
44
44
  [![PyPI](https://img.shields.io/pypi/v/rustling.svg)](https://pypi.org/project/rustling/)
45
- [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
46
45
  [![Conda Version](https://img.shields.io/conda/vn/conda-forge/rustling.svg)](https://anaconda.org/conda-forge/rustling)
46
+ [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
47
47
 
48
48
  Rustling is a blazingly fast library for computational linguistics.
49
+ It aims to provide flexible and efficient tools to facilitate further research.
49
50
 
50
51
  Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
51
52
 
52
- ## Features
53
+ Currently implemented features:
54
+
55
+ * Sequence modeling:
56
+
57
+ - N-grams and related language models
58
+ - Hidden Markov model
59
+ - Word segmentation
60
+ - Averaged perceptron part-of-speech tagging
61
+
62
+ * Handling richly formatted data,
63
+ supporting cross-format conversion as well as both local and remote sources for data ingestion:
53
64
 
54
- - N-grams
55
- - Language models
56
- - Hidden Markov model
57
- - Word segmentation
58
- - Part-of-speech tagging
59
- - CHAT parsing for TalkBank and CHILDES data
65
+ - CHAT for TalkBank and CHILDES
66
+ - ELAN for annotated multimedia data
67
+ - TextGrid for Praat annotations
68
+ - CoNLL-U for University Dependencies
69
+ - SRT for SubRip subtitles
60
70
 
61
71
  ## Performance
62
72
 
63
- | Component | Task | Speedup | vs. |
64
- |---|---|---|---|
65
- | **Language Models** | Fit | **11x** | NLTK |
66
- | | Score | **2x** | NLTK |
67
- | | Generate | **86--107x** | NLTK |
68
- | **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
69
- | **POS Tagging** | Training | **5x** | NLTK |
70
- | | Tagging | **17x** | NLTK |
71
- | **HMM** | Fit | **14x** | hmmlearn |
72
- | | Predict | **0.9x** | hmmlearn |
73
- | | Score | **5x** | hmmlearn |
74
- | **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
75
- | | Reading from strings | **35x** | pylangacq |
76
- | | Parsing utterances | **15x** | pylangacq |
77
- | | Parsing tokens | **8x** | pylangacq |
78
-
79
- See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
73
+ Rustling is highly performant because it is implemented in Rust under the hood.
74
+ For benchmarks comparing Rustling against other Python packages with similar functionalities,
75
+ please see [`benchmarks`](https://github.com/jacksonllee/rustling/tree/main/benchmarks).
80
76
 
81
77
 
82
78
  ## Installation
@@ -6,41 +6,37 @@
6
6
  <br>
7
7
 
8
8
  [![PyPI](https://img.shields.io/pypi/v/rustling.svg)](https://pypi.org/project/rustling/)
9
- [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
10
9
  [![Conda Version](https://img.shields.io/conda/vn/conda-forge/rustling.svg)](https://anaconda.org/conda-forge/rustling)
10
+ [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
11
11
 
12
12
  Rustling is a blazingly fast library for computational linguistics.
13
+ It aims to provide flexible and efficient tools to facilitate further research.
13
14
 
14
15
  Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
15
16
 
16
- ## Features
17
+ Currently implemented features:
18
+
19
+ * Sequence modeling:
20
+
21
+ - N-grams and related language models
22
+ - Hidden Markov model
23
+ - Word segmentation
24
+ - Averaged perceptron part-of-speech tagging
25
+
26
+ * Handling richly formatted data,
27
+ supporting cross-format conversion as well as both local and remote sources for data ingestion:
17
28
 
18
- - N-grams
19
- - Language models
20
- - Hidden Markov model
21
- - Word segmentation
22
- - Part-of-speech tagging
23
- - CHAT parsing for TalkBank and CHILDES data
29
+ - CHAT for TalkBank and CHILDES
30
+ - ELAN for annotated multimedia data
31
+ - TextGrid for Praat annotations
32
+ - CoNLL-U for University Dependencies
33
+ - SRT for SubRip subtitles
24
34
 
25
35
  ## Performance
26
36
 
27
- | Component | Task | Speedup | vs. |
28
- |---|---|---|---|
29
- | **Language Models** | Fit | **11x** | NLTK |
30
- | | Score | **2x** | NLTK |
31
- | | Generate | **86--107x** | NLTK |
32
- | **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
33
- | **POS Tagging** | Training | **5x** | NLTK |
34
- | | Tagging | **17x** | NLTK |
35
- | **HMM** | Fit | **14x** | hmmlearn |
36
- | | Predict | **0.9x** | hmmlearn |
37
- | | Score | **5x** | hmmlearn |
38
- | **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
39
- | | Reading from strings | **35x** | pylangacq |
40
- | | Parsing utterances | **15x** | pylangacq |
41
- | | Parsing tokens | **8x** | pylangacq |
42
-
43
- See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
37
+ Rustling is highly performant because it is implemented in Rust under the hood.
38
+ For benchmarks comparing Rustling against other Python packages with similar functionalities,
39
+ please see [`benchmarks`](https://github.com/jacksonllee/rustling/tree/main/benchmarks).
44
40
 
45
41
 
46
42
  ## Installation
@@ -0,0 +1,132 @@
1
+ # Benchmarks
2
+
3
+ This directory contains benchmarking scripts to compare Rustling (Rust + PyO3) against other Python packages with similar functionalities.
4
+
5
+ **GitHub**: https://github.com/jacksonllee/rustling/tree/main/benchmarks
6
+
7
+ ## Directory Structure
8
+
9
+ ```
10
+ benchmarks/
11
+ ├── README.md
12
+ ├── run_chat.py # CHAT parsing benchmark (Rustling vs pylangacq)
13
+ ├── run_conllu.py # CoNLL-U parsing benchmark (Rustling vs conllu)
14
+ ├── run_elan.py # ELAN parsing benchmark (Rustling vs pympi-ling)
15
+ ├── run_textgrid.py # TextGrid parsing benchmark (Rustling vs pympi-ling)
16
+ ├── run_hmm.py # HMM benchmark (Rustling vs hmmlearn)
17
+ ├── run_lm.py # Language model benchmark (Rustling vs NLTK)
18
+ ├── run_wordseg.py # Word segmentation benchmark (Rustling vs wordseg)
19
+ ├── run_perceptron_pos_tagger.py # POS tagger benchmark (Rustling vs NLTK PerceptronTagger)
20
+ ├── update_readme.py # Update benchmark tables in README files
21
+ └── common/
22
+ ├── __init__.py
23
+ └── data.py # Shared HKCanCor data loader
24
+ ```
25
+
26
+ ## Data Sources
27
+
28
+ Most benchmarks use the **HKCanCor** corpus (~10K Cantonese sentences with POS tags), loaded via pycantonese. The shared data loader in `common/data.py` converts the corpus into the format each benchmark needs:
29
+
30
+ - **Tagging**: tagged sentences `[(word, tag), ...]` for training, untagged word lists for testing
31
+ - **Word segmentation**: word tuples for training, concatenated strings for testing
32
+ - **HMM**: word sequences (tags stripped) for unsupervised Baum-Welch EM training and Viterbi decoding
33
+ - **Language models**: word sequences (tags stripped)
34
+
35
+ The CoNLL-U benchmark uses the **UD_English-EWT** treebank (English Universal Dependencies data), auto-downloaded to `~/.rustling/ud-english-ewt/`.
36
+
37
+ The ELAN benchmark uses the **CantoMap** corpus (Cantonese conversation data with ELAN annotations), auto-downloaded to `~/.rustling/cantomap/`.
38
+
39
+ The TextGrid benchmark uses TextGrid files generated from the CantoMap ELAN data via `rustling.elan.ELAN.to_textgrid_files()`, cached at `~/.rustling/cantomap_textgrid/`.
40
+
41
+ ## Prerequisites
42
+
43
+ ```bash
44
+ # Build Rustling (from repo root)
45
+ uv run maturin develop --release
46
+
47
+ # Install benchmark dependencies
48
+ uv sync --group benchmarks
49
+ ```
50
+
51
+ ### Comparison Libraries
52
+
53
+ | Benchmark | Comparison Library |
54
+ |-----------|--------------------|
55
+ | CHAT Parsing | [pylangacq](https://pylangacq.org/) |
56
+ | CoNLL-U Parsing | [conllu](https://github.com/EmilStenstrom/conllu/) |
57
+ | ELAN Parsing | [pympi-ling](https://pypi.org/project/pympi-ling/) Eaf |
58
+ | TextGrid Parsing | [pympi-ling](https://pypi.org/project/pympi-ling/) TextGrid |
59
+ | HMM | [hmmlearn](https://hmmlearn.readthedocs.io/) CategoricalHMM |
60
+ | Word Segmentation | [wordseg](https://pypi.org/project/wordseg/) |
61
+ | POS Tagging | [NLTK](https://www.nltk.org/) PerceptronTagger |
62
+ | Language Models | [NLTK](https://www.nltk.org/) nltk.lm |
63
+
64
+ All benchmarks degrade gracefully if a comparison library is not installed.
65
+
66
+ ## Results
67
+
68
+ Benchmarked against Python implementations from NLTK, wordseg (v0.0.5),
69
+ pylangacq (v0.19.1), hmmlearn (v0.3.3), pympi-ling (v1.70.2), and conllu (v6.0.0).
70
+
71
+ | Component | Task | Speedup | vs. |
72
+ |---|---|---|---|
73
+ | **Language Models** | Fit | **11x** | NLTK |
74
+ | | Score | **2x** | NLTK |
75
+ | | Generate | **86--107x** | NLTK |
76
+ | **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
77
+ | **POS Tagging** | Training | **5x** | NLTK |
78
+ | | Tagging | **17x** | NLTK |
79
+ | **HMM** | Fit | **14x** | hmmlearn |
80
+ | | Predict | **0.9x** | hmmlearn |
81
+ | | Score | **5x** | hmmlearn |
82
+ | **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
83
+ | | Reading from strings | **35x** | pylangacq |
84
+ | | Parsing utterances | **15x** | pylangacq |
85
+ | | Parsing tokens | **8x** | pylangacq |
86
+ | **ELAN Parsing** | Parse single file | **4x** | pympi-ling |
87
+ | | Parse all files | **17x** | pympi-ling |
88
+ | **TextGrid Parsing** | Parse single file | **3x** | pympi-ling |
89
+ | | Parse all files | **8x** | pympi-ling |
90
+ | **CoNLL-U Parsing** | Parse from strings | **15x** | conllu |
91
+ | | Parse from files | **15x** | conllu |
92
+
93
+ ---
94
+
95
+ ## Running Benchmarks
96
+
97
+ Each script supports `--quick` (fewer iterations), `--export FILE` (JSON output), and `--quiet`:
98
+
99
+ ```bash
100
+ python benchmarks/run_chat.py
101
+ python benchmarks/run_conllu.py
102
+ python benchmarks/run_elan.py
103
+ python benchmarks/run_textgrid.py
104
+ python benchmarks/run_hmm.py
105
+ python benchmarks/run_wordseg.py
106
+ python benchmarks/run_perceptron_pos_tagger.py
107
+ python benchmarks/run_lm.py
108
+ ```
109
+
110
+ ## Updating Benchmark Tables
111
+
112
+ After running benchmarks with `--export`, update the performance table in `benchmarks/README.md`:
113
+
114
+ ```bash
115
+ python benchmarks/run_chat.py --export benchmarks/.results/chat.json
116
+ python benchmarks/run_conllu.py --export benchmarks/.results/conllu.json
117
+ python benchmarks/run_elan.py --export benchmarks/.results/elan.json
118
+ python benchmarks/run_textgrid.py --export benchmarks/.results/textgrid.json
119
+ python benchmarks/run_hmm.py --export benchmarks/.results/hmm.json
120
+ python benchmarks/run_wordseg.py --export benchmarks/.results/wordseg.json
121
+ python benchmarks/run_perceptron_pos_tagger.py --export benchmarks/.results/tagger.json
122
+ python benchmarks/run_lm.py --export benchmarks/.results/lm.json
123
+
124
+ python benchmarks/update_readme.py --from-json benchmarks/.results/
125
+ ```
126
+
127
+ ## Tips
128
+
129
+ - Use `--release` when building Rustling for accurate benchmarks: `maturin develop --release`
130
+ - Close other applications to reduce noise
131
+ - Run multiple times to verify consistency
132
+ - Use `--quiet` with `--export` for machine-readable output only
@@ -0,0 +1,300 @@
1
+ #!/usr/bin/env python
2
+ """Benchmark rustling.conllu vs conllu for CoNLL-U parsing.
3
+
4
+ Compares CoNLL-U file loading speed using UD_English-EWT treebank data.
5
+
6
+ Usage:
7
+ python benchmarks/run_conllu.py
8
+ python benchmarks/run_conllu.py --quick
9
+ python benchmarks/run_conllu.py --export results.json
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import gc
16
+ import json
17
+ import subprocess
18
+ import sys
19
+ import time
20
+ from dataclasses import dataclass, field
21
+ from pathlib import Path
22
+ from typing import Any, Callable
23
+
24
+ UD_ENGLISH_EWT_DIR = Path.home() / ".rustling" / "ud-english-ewt"
25
+
26
+
27
+ @dataclass
28
+ class BenchmarkResult:
29
+ """Result of a single benchmark run."""
30
+
31
+ name: str
32
+ implementation: str
33
+ time_seconds: float
34
+ iterations: int
35
+ detail: str = ""
36
+ ops_per_second: float = field(init=False, default=0.0)
37
+
38
+ def __post_init__(self) -> None:
39
+ if self.time_seconds > 0:
40
+ self.ops_per_second = self.iterations / self.time_seconds
41
+
42
+
43
+ def ensure_ud_english_ewt() -> Path:
44
+ """Download UD_English-EWT data if not present."""
45
+ if not UD_ENGLISH_EWT_DIR.exists():
46
+ print("Downloading UD_English-EWT data...")
47
+ UD_ENGLISH_EWT_DIR.mkdir(parents=True, exist_ok=True)
48
+ subprocess.run(
49
+ [
50
+ "git",
51
+ "clone",
52
+ "--depth",
53
+ "1",
54
+ "https://github.com/UniversalDependencies/UD_English-EWT.git",
55
+ str(UD_ENGLISH_EWT_DIR),
56
+ ],
57
+ check=True,
58
+ )
59
+ return UD_ENGLISH_EWT_DIR
60
+
61
+
62
+ def collect_conllu_files(ud_dir: Path) -> list[Path]:
63
+ """Collect all .conllu files from the treebank directory."""
64
+ conllu_files = sorted(ud_dir.glob("*.conllu"))
65
+ if not conllu_files:
66
+ print("Error: No .conllu files found in UD_English-EWT data.")
67
+ sys.exit(1)
68
+ return conllu_files
69
+
70
+
71
+ def time_function(
72
+ func: Callable[[], Any],
73
+ iterations: int = 5,
74
+ warmup: int = 1,
75
+ ) -> float:
76
+ """Time a function over multiple iterations.
77
+
78
+ Returns
79
+ -------
80
+ float
81
+ Total time in seconds.
82
+ """
83
+ for _ in range(warmup):
84
+ func()
85
+
86
+ gc.collect()
87
+
88
+ times = []
89
+ for _ in range(iterations):
90
+ start = time.perf_counter()
91
+ func()
92
+ times.append(time.perf_counter() - start)
93
+
94
+ return sum(times)
95
+
96
+
97
+ def print_result(result: BenchmarkResult) -> None:
98
+ """Print a benchmark result."""
99
+ print(f" {result.implementation}:")
100
+ print(
101
+ f" Total time: {result.time_seconds:.4f}s ({result.iterations} iterations)"
102
+ )
103
+ if result.detail:
104
+ print(f" Detail: {result.detail}")
105
+
106
+
107
+ def print_comparison(
108
+ rustling_result: BenchmarkResult, python_result: BenchmarkResult
109
+ ) -> None:
110
+ """Print comparison between rustling and conllu."""
111
+ if rustling_result.time_seconds > 0:
112
+ speedup = python_result.time_seconds / rustling_result.time_seconds
113
+ else:
114
+ speedup = float("inf")
115
+ print(f"\n Speedup: {speedup:.1f}x faster")
116
+
117
+
118
+ def run_benchmarks(
119
+ quick: bool = False,
120
+ verbose: bool = True,
121
+ ) -> dict[str, Any]:
122
+ """Run all benchmarks."""
123
+ ud_dir = ensure_ud_english_ewt()
124
+ conllu_files = collect_conllu_files(ud_dir)
125
+ conllu_paths_str = [str(p) for p in conllu_files]
126
+
127
+ rustling_conllu = None
128
+ conllu_pkg = None
129
+
130
+ try:
131
+ from rustling.conllu import CoNLLU
132
+
133
+ rustling_conllu = CoNLLU
134
+ if verbose:
135
+ print("rustling.conllu loaded successfully")
136
+ except ImportError as e:
137
+ if verbose:
138
+ print(f"rustling.conllu not available: {e}")
139
+
140
+ try:
141
+ import conllu as _conllu_pkg
142
+
143
+ conllu_pkg = _conllu_pkg
144
+ if verbose:
145
+ print("conllu loaded successfully")
146
+ except ImportError as e:
147
+ if verbose:
148
+ print(f"conllu not available: {e}")
149
+
150
+ if rustling_conllu is None and conllu_pkg is None:
151
+ print("\nError: Neither implementation is available.")
152
+ sys.exit(1)
153
+
154
+ iterations = 3 if quick else 10
155
+
156
+ all_results: dict[str, Any] = {"benchmarks": {}}
157
+
158
+ # Pre-read file contents for from_strs benchmark
159
+ conllu_strs = []
160
+ for f in conllu_files:
161
+ conllu_strs.append(f.read_text(encoding="utf-8"))
162
+
163
+ print("\n" + "=" * 60)
164
+ print("CONLLU BENCHMARK: Rustling (Rust) vs conllu (Python)")
165
+ print(f"Dataset: UD_English-EWT ({len(conllu_files)} .conllu files)")
166
+ print("=" * 60)
167
+
168
+ # Benchmark 1: from_strs (parse from in-memory strings)
169
+ if verbose:
170
+ print(f"\nfrom_strs (parsing {len(conllu_strs)} in-memory strings):")
171
+
172
+ results = []
173
+ for name, impl in [("rustling", rustling_conllu), ("conllu", conllu_pkg)]:
174
+ if impl is None:
175
+ results.append(None)
176
+ continue
177
+
178
+ if name == "rustling":
179
+
180
+ def parse_strs() -> None:
181
+ impl.from_strs(conllu_strs)
182
+
183
+ else:
184
+
185
+ def parse_strs() -> None:
186
+ for s in conllu_strs:
187
+ impl.parse(s)
188
+
189
+ total_time = time_function(parse_strs, iterations=iterations)
190
+ result = BenchmarkResult(
191
+ name="from_strs",
192
+ implementation=name,
193
+ time_seconds=total_time,
194
+ iterations=iterations,
195
+ )
196
+ results.append(result)
197
+ if verbose:
198
+ print_result(result)
199
+
200
+ if results[0] and results[1] and verbose:
201
+ print_comparison(results[0], results[1])
202
+
203
+ all_results["benchmarks"]["from_strs"] = {
204
+ "rustling": results[0].__dict__ if results[0] else None,
205
+ "conllu": results[1].__dict__ if results[1] else None,
206
+ }
207
+
208
+ # Benchmark 2: from_files (parse from disk)
209
+ if verbose:
210
+ print(f"\nfrom_files (parsing {len(conllu_files)} files from disk):")
211
+
212
+ results = []
213
+ for name, impl in [("rustling", rustling_conllu), ("conllu", conllu_pkg)]:
214
+ if impl is None:
215
+ results.append(None)
216
+ continue
217
+
218
+ if name == "rustling":
219
+
220
+ def parse_files() -> None:
221
+ impl.from_files(conllu_paths_str)
222
+
223
+ else:
224
+
225
+ def parse_files() -> None:
226
+ for p in conllu_paths_str:
227
+ with open(p, encoding="utf-8") as f:
228
+ impl.parse(f.read())
229
+
230
+ total_time = time_function(parse_files, iterations=iterations)
231
+ result = BenchmarkResult(
232
+ name="from_files",
233
+ implementation=name,
234
+ time_seconds=total_time,
235
+ iterations=iterations,
236
+ detail=f"{len(conllu_files)} files",
237
+ )
238
+ results.append(result)
239
+ if verbose:
240
+ print_result(result)
241
+
242
+ if results[0] and results[1] and verbose:
243
+ print_comparison(results[0], results[1])
244
+
245
+ all_results["benchmarks"]["from_files"] = {
246
+ "rustling": results[0].__dict__ if results[0] else None,
247
+ "conllu": results[1].__dict__ if results[1] else None,
248
+ }
249
+
250
+ # Compute speedups for each task
251
+ speedups: dict[str, float] = {}
252
+ for task in ["from_strs", "from_files"]:
253
+ bench = all_results["benchmarks"].get(task, {})
254
+ r = bench.get("rustling")
255
+ p = bench.get("conllu")
256
+ if r and p and r["time_seconds"] > 0:
257
+ speedups[task] = p["time_seconds"] / r["time_seconds"]
258
+ all_results["speedups"] = speedups
259
+
260
+ return all_results
261
+
262
+
263
+ def main() -> None:
264
+ """Main entry point."""
265
+ parser = argparse.ArgumentParser(
266
+ description="Benchmark rustling.conllu vs conllu for CoNLL-U parsing"
267
+ )
268
+ parser.add_argument(
269
+ "--quick",
270
+ action="store_true",
271
+ help="Run quick benchmark with fewer iterations",
272
+ )
273
+ parser.add_argument(
274
+ "--export",
275
+ type=str,
276
+ metavar="FILE",
277
+ help="Export results to JSON file",
278
+ )
279
+ parser.add_argument(
280
+ "--quiet",
281
+ action="store_true",
282
+ help="Suppress output (useful with --export)",
283
+ )
284
+
285
+ args = parser.parse_args()
286
+
287
+ results = run_benchmarks(
288
+ quick=args.quick,
289
+ verbose=not args.quiet,
290
+ )
291
+
292
+ if args.export:
293
+ export_path = Path(args.export)
294
+ with open(export_path, "w") as f:
295
+ json.dump(results, f, indent=2)
296
+ print(f"\nResults exported to: {export_path}")
297
+
298
+
299
+ if __name__ == "__main__":
300
+ main()