rustling 0.6.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. rustling-0.7.0/.github/workflows/build-wheel.yml +42 -0
  2. {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/python.yml +1 -1
  3. {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/release.yml +2 -1
  4. {rustling-0.6.0 → rustling-0.7.0}/.pre-commit-config.yaml +11 -1
  5. rustling-0.7.0/CHANGELOG.md +70 -0
  6. {rustling-0.6.0 → rustling-0.7.0}/Cargo.lock +11 -11
  7. {rustling-0.6.0 → rustling-0.7.0}/Cargo.toml +4 -15
  8. {rustling-0.6.0 → rustling-0.7.0}/PKG-INFO +31 -17
  9. rustling-0.7.0/README.md +74 -0
  10. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/README.md +1 -1
  11. {rustling-0.6.0 → rustling-0.7.0}/pyproject.toml +1 -4
  12. {rustling-0.6.0 → rustling-0.7.0}/python/docs/index.rst +25 -12
  13. {rustling-0.6.0 → rustling-0.7.0}/python/docs/wordseg.rst +8 -8
  14. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/chat/__init__.pyi +49 -4
  15. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/wordseg/__init__.pyi +113 -9
  16. {rustling-0.6.0 → rustling-0.7.0}/python/stubtest_allowlist.txt +4 -0
  17. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_chat/test_chat.py +146 -14
  18. rustling-0.7.0/python/tests/test_wordseg/test_dag_hmm_segmenter.py +69 -0
  19. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_hmm_segmenter.py +42 -0
  20. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_longest_string_matching.py +36 -0
  21. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_random_segmenter.py +31 -0
  22. {rustling-0.6.0 → rustling-0.7.0}/src/chat/clean_utterance.rs +481 -11
  23. {rustling-0.6.0 → rustling-0.7.0}/src/chat/header.rs +85 -140
  24. rustling-0.7.0/src/chat/header_py.rs +312 -0
  25. {rustling-0.6.0 → rustling-0.7.0}/src/chat/ipsyn.rs +4 -0
  26. {rustling-0.6.0 → rustling-0.7.0}/src/chat/mod.rs +16 -5
  27. {rustling-0.6.0 → rustling-0.7.0}/src/chat/reader.rs +307 -1128
  28. rustling-0.7.0/src/chat/reader_py.rs +1128 -0
  29. {rustling-0.6.0 → rustling-0.7.0}/src/chat/utterance.rs +108 -309
  30. rustling-0.7.0/src/chat/utterance_py.rs +330 -0
  31. rustling-0.7.0/src/chat/validation.rs +1091 -0
  32. {rustling-0.6.0 → rustling-0.7.0}/src/hmm/mod.rs +52 -276
  33. rustling-0.7.0/src/hmm/py.rs +254 -0
  34. {rustling-0.6.0 → rustling-0.7.0}/src/lib.rs +4 -1
  35. {rustling-0.6.0 → rustling-0.7.0}/src/lm/mod.rs +40 -382
  36. rustling-0.7.0/src/lm/py.rs +370 -0
  37. rustling-0.6.0/src/ngram.rs → rustling-0.7.0/src/ngram/mod.rs +30 -222
  38. rustling-0.7.0/src/ngram/py.rs +204 -0
  39. {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/mod.rs +8 -141
  40. rustling-0.7.0/src/perceptron_pos_tagger/py.rs +144 -0
  41. {rustling-0.6.0 → rustling-0.7.0}/src/persistence.rs +28 -2
  42. rustling-0.7.0/src/prelude.rs +15 -0
  43. rustling-0.6.0/src/seq_feature.rs → rustling-0.7.0/src/seq_feature/mod.rs +23 -80
  44. rustling-0.7.0/src/seq_feature/py.rs +74 -0
  45. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/dag_hmm.rs +45 -60
  46. rustling-0.7.0/src/wordseg/dag_hmm_py.rs +76 -0
  47. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/hmm.rs +86 -83
  48. rustling-0.7.0/src/wordseg/hmm_py.rs +99 -0
  49. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/mod.rs +15 -100
  50. rustling-0.7.0/src/wordseg/longest_string_matching/py.rs +109 -0
  51. rustling-0.7.0/src/wordseg/mod.rs +141 -0
  52. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/random_segmenter.rs +12 -59
  53. rustling-0.7.0/src/wordseg/random_segmenter_py.rs +68 -0
  54. rustling-0.6.0/AGENTS.md +0 -335
  55. rustling-0.6.0/CHANGELOG.md +0 -42
  56. rustling-0.6.0/CLAUDE.md +0 -335
  57. rustling-0.6.0/README.md +0 -57
  58. rustling-0.6.0/src/wordseg/mod.rs +0 -55
  59. {rustling-0.6.0 → rustling-0.7.0}/.flake8 +0 -0
  60. {rustling-0.6.0 → rustling-0.7.0}/.gitattributes +0 -0
  61. {rustling-0.6.0 → rustling-0.7.0}/.github/dependabot.yml +0 -0
  62. {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/rust.yml +0 -0
  63. {rustling-0.6.0 → rustling-0.7.0}/.gitignore +0 -0
  64. {rustling-0.6.0 → rustling-0.7.0}/.readthedocs.yaml +0 -0
  65. {rustling-0.6.0 → rustling-0.7.0}/LICENSE.md +0 -0
  66. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/common/__init__.py +0 -0
  67. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/common/data.py +0 -0
  68. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_chat.py +0 -0
  69. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_hmm.py +0 -0
  70. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_lm.py +0 -0
  71. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_perceptron_pos_tagger.py +0 -0
  72. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_wordseg.py +0 -0
  73. {rustling-0.6.0 → rustling-0.7.0}/benchmarks/update_docs.py +0 -0
  74. {rustling-0.6.0 → rustling-0.7.0}/build.rs +0 -0
  75. {rustling-0.6.0 → rustling-0.7.0}/python/docs/Makefile +0 -0
  76. {rustling-0.6.0 → rustling-0.7.0}/python/docs/README.md +0 -0
  77. {rustling-0.6.0 → rustling-0.7.0}/python/docs/_static/favicon.ico +0 -0
  78. {rustling-0.6.0 → rustling-0.7.0}/python/docs/_static/logo-with-text.svg +0 -0
  79. {rustling-0.6.0 → rustling-0.7.0}/python/docs/_templates/copyright.html +0 -0
  80. {rustling-0.6.0 → rustling-0.7.0}/python/docs/api.rst +0 -0
  81. {rustling-0.6.0 → rustling-0.7.0}/python/docs/chat.rst +0 -0
  82. {rustling-0.6.0 → rustling-0.7.0}/python/docs/conf.py +0 -0
  83. {rustling-0.6.0 → rustling-0.7.0}/python/docs/hmm.rst +0 -0
  84. {rustling-0.6.0 → rustling-0.7.0}/python/docs/lm.rst +0 -0
  85. {rustling-0.6.0 → rustling-0.7.0}/python/docs/make.bat +0 -0
  86. {rustling-0.6.0 → rustling-0.7.0}/python/docs/ngram.rst +0 -0
  87. {rustling-0.6.0 → rustling-0.7.0}/python/docs/perceptron_pos_tagger.rst +0 -0
  88. {rustling-0.6.0 → rustling-0.7.0}/python/docs/requirements.txt +0 -0
  89. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/__init__.py +0 -0
  90. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/chat/__init__.py +0 -0
  91. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/hmm/__init__.py +0 -0
  92. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/hmm/__init__.pyi +0 -0
  93. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/lm/__init__.py +0 -0
  94. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/lm/__init__.pyi +0 -0
  95. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/ngram/__init__.py +0 -0
  96. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/ngram/__init__.pyi +0 -0
  97. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/perceptron_pos_tagger/__init__.py +0 -0
  98. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/perceptron_pos_tagger/__init__.pyi +0 -0
  99. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/py.typed +0 -0
  100. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/seq_feature/__init__.py +0 -0
  101. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/seq_feature/__init__.pyi +0 -0
  102. {rustling-0.6.0 → rustling-0.7.0}/python/rustling/wordseg/__init__.py +0 -0
  103. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_benchmarks.py +0 -0
  104. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_chat/conftest.py +0 -0
  105. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_hmm.py +0 -0
  106. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_lm.py +0 -0
  107. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_ngram.py +0 -0
  108. {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_perceptron_pos_tagger.py +0 -0
  109. {rustling-0.6.0 → rustling-0.7.0}/src/hmm/model.fbs +0 -0
  110. {rustling-0.6.0 → rustling-0.7.0}/src/hmm/model_generated.rs +0 -0
  111. {rustling-0.6.0 → rustling-0.7.0}/src/lm/model.fbs +0 -0
  112. {rustling-0.6.0 → rustling-0.7.0}/src/lm/model_generated.rs +0 -0
  113. {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/model.fbs +0 -0
  114. {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/model_generated.rs +0 -0
  115. {rustling-0.6.0 → rustling-0.7.0}/src/trie.rs +0 -0
  116. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/bmes.rs +0 -0
  117. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/model.fbs +0 -0
  118. {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/model_generated.rs +0 -0
@@ -0,0 +1,42 @@
1
+ name: Build Linux x86_64 wheel
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ git_ref:
7
+ description: "Git ref to build from (tag, branch, or commit)"
8
+ required: true
9
+ default: "v0.6.0"
10
+
11
+ env:
12
+ FLATC_VERSION: "25.12.19"
13
+
14
+ jobs:
15
+ build:
16
+ name: Build manylinux x86_64 wheel
17
+ runs-on: ubuntu-latest
18
+ steps:
19
+ - uses: actions/checkout@v6
20
+ with:
21
+ ref: ${{ github.event.inputs.git_ref }}
22
+
23
+ - name: Build wheels
24
+ uses: PyO3/maturin-action@v1
25
+ with:
26
+ target: x86_64
27
+ args: --release --out dist -i python3.10
28
+ sccache: "true"
29
+ manylinux: auto
30
+ before-script-linux: |
31
+ curl -sL "https://github.com/google/flatbuffers/releases/download/v${{ env.FLATC_VERSION }}/Linux.flatc.binary.clang++-18.zip" -o flatc.zip
32
+ command -v unzip || yum install -y unzip 2>/dev/null || (apt-get update -qq && apt-get install -y -qq unzip)
33
+ unzip -q flatc.zip
34
+ rm flatc.zip
35
+ chmod +x flatc
36
+ mv flatc /usr/local/bin/
37
+
38
+ - name: Upload wheel
39
+ uses: actions/upload-artifact@v7
40
+ with:
41
+ name: wheel-linux-x86_64
42
+ path: dist
@@ -146,7 +146,7 @@ jobs:
146
146
  uses: PyO3/maturin-action@v1
147
147
  with:
148
148
  target: ${{ matrix.target }}
149
- args: --release --out dist
149
+ args: --release --out dist -i python3.10
150
150
  sccache: "true"
151
151
  manylinux: auto
152
152
  before-script-linux: |
@@ -71,7 +71,7 @@ jobs:
71
71
  uses: PyO3/maturin-action@v1
72
72
  with:
73
73
  target: ${{ matrix.target }}
74
- args: --release --out dist
74
+ args: --release --out dist -i python3.10
75
75
  sccache: "true"
76
76
  manylinux: auto
77
77
  before-script-linux: |
@@ -201,6 +201,7 @@ jobs:
201
201
  run: |
202
202
  curl -sL "https://github.com/google/flatbuffers/releases/download/v${{ env.FLATC_VERSION }}/Linux.flatc.binary.clang++-18.zip" -o flatc.zip
203
203
  unzip -q flatc.zip
204
+ rm flatc.zip
204
205
  chmod +x flatc
205
206
  sudo mv flatc /usr/local/bin/
206
207
 
@@ -14,7 +14,7 @@ repos:
14
14
  types: [rust]
15
15
  pass_filenames: false
16
16
  - repo: https://github.com/psf/black-pre-commit-mirror
17
- rev: 26.1.0
17
+ rev: 26.3.0
18
18
  hooks:
19
19
  - id: black
20
20
  files: ^(python|benchmarks)/
@@ -23,3 +23,13 @@ repos:
23
23
  hooks:
24
24
  - id: flake8
25
25
  files: ^(python|benchmarks)/
26
+ - repo: local
27
+ hooks:
28
+ - id: mypy
29
+ name: mypy
30
+ entry: uvx mypy
31
+ language: system
32
+ types: [python]
33
+ files: ^python/rustling/
34
+ pass_filenames: false
35
+ args: [python/rustling/]
@@ -0,0 +1,70 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.7.0] - 2026-03-14
11
+
12
+ ### Added
13
+
14
+ - Word segmentation:
15
+ * Added `score` method for the HMM and DAG-HMM segmenters.
16
+ * `predict` method can optionally output offsets for the (start, end) indices
17
+ of segmented words compared to the original string.
18
+ - CHAT parsing: Support custom tier names other than the standard %mor and %gra.
19
+ - Python model classes are now subclassable.
20
+
21
+ ### Changed
22
+
23
+ - Ngram counters: `Ngrams.most_common` now sorts tuples lexicographically
24
+ when counts are tied.
25
+ - CHAT parsing:
26
+ * If a date is available at `Headers`'s `date`,
27
+ it's now a Python `datetime.date` object instead of a string.
28
+ * In handling the main tier transcription for creating `Token` objects:
29
+ - Special form markers suffixed with "@" are now stripped.
30
+ - Words that have partiallly parenthetical material have the parentheses
31
+ removed, e.g., (un)til -> until, sit(ting) -> sitting.
32
+ * Renamed the `CHAT.raw` attribute to `CHAT.audible` for a best-effort,
33
+ audibly faithful transcription string, to facilitate automatic speech recognition,
34
+ forced alignment, etc.
35
+ * A subset of the testchat/bad dataset is now used to validate CHAT data format.
36
+ - Refactored core Rust code so that Rust-only consumers no longer need PyO3/Python.
37
+
38
+ ## [0.6.0] - 2026-03-05
39
+
40
+ ### Added
41
+
42
+ - Hidden Markov Model (HMM)
43
+ - Word segmentation: Added DAG-HMM word segmenter
44
+ - CHAT parsing: Added `from_utterances` method
45
+
46
+ ### Changed
47
+
48
+ - Models are now persisted as a zstd-compressed FlatBuffers binary.
49
+
50
+ ## [0.5.0] - 2026-02-18
51
+
52
+ ### Added
53
+
54
+ - CHAT parsing for TalkBank and CHILDES data
55
+
56
+ ## [0.4.0] - 2026-02-08
57
+
58
+ ### Added
59
+
60
+ - N-grams and language models
61
+
62
+ ## [0.3.0] - 2026-02-06
63
+
64
+ ### Added
65
+
66
+ - Averaged perceptron tagger
67
+
68
+ ## [0.2.0] - 2026-02-04
69
+
70
+ - Initial release, with longest string matching and random segmenter for word segmentation
@@ -81,9 +81,9 @@ dependencies = [
81
81
 
82
82
  [[package]]
83
83
  name = "cc"
84
- version = "1.2.56"
84
+ version = "1.2.57"
85
85
  source = "registry+https://github.com/rust-lang/crates.io-index"
86
- checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
86
+ checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
87
87
  dependencies = [
88
88
  "find-msvc-tools",
89
89
  "jobserver",
@@ -418,9 +418,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
418
418
 
419
419
  [[package]]
420
420
  name = "libc"
421
- version = "0.2.182"
421
+ version = "0.2.183"
422
422
  source = "registry+https://github.com/rust-lang/crates.io-index"
423
- checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
423
+ checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
424
424
 
425
425
  [[package]]
426
426
  name = "linux-raw-sys"
@@ -467,9 +467,9 @@ checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
467
467
 
468
468
  [[package]]
469
469
  name = "once_cell"
470
- version = "1.21.3"
470
+ version = "1.21.4"
471
471
  source = "registry+https://github.com/rust-lang/crates.io-index"
472
- checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
472
+ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
473
473
 
474
474
  [[package]]
475
475
  name = "pbkdf2"
@@ -699,7 +699,7 @@ dependencies = [
699
699
 
700
700
  [[package]]
701
701
  name = "rustling"
702
- version = "0.6.0"
702
+ version = "0.7.0"
703
703
  dependencies = [
704
704
  "fancy-regex",
705
705
  "flatbuffers",
@@ -840,9 +840,9 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
840
840
 
841
841
  [[package]]
842
842
  name = "tempfile"
843
- version = "3.26.0"
843
+ version = "3.27.0"
844
844
  source = "registry+https://github.com/rust-lang/crates.io-index"
845
- checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
845
+ checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
846
846
  dependencies = [
847
847
  "fastrand",
848
848
  "getrandom 0.4.2",
@@ -897,9 +897,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
897
897
 
898
898
  [[package]]
899
899
  name = "uuid"
900
- version = "1.21.0"
900
+ version = "1.22.0"
901
901
  source = "registry+https://github.com/rust-lang/crates.io-index"
902
- checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
902
+ checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
903
903
  dependencies = [
904
904
  "getrandom 0.4.2",
905
905
  "js-sys",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "rustling"
3
- version = "0.6.0"
3
+ version = "0.7.0"
4
4
  edition = "2024"
5
5
  description = "A blazingly fast library for computational linguistics"
6
6
  license = "MIT"
@@ -8,21 +8,10 @@ repository = "https://github.com/jacksonllee/rustling"
8
8
  documentation = "https://docs.rs/rustling"
9
9
  readme = "README.md"
10
10
  keywords = [
11
- "computational-linguistics",
11
+ # At most 5 keywords per crate. Each keyword must have fewer than 20 characters.
12
12
  "linguistics",
13
- "natural-language-processing",
14
13
  "nlp",
15
14
  "text-processing",
16
- "word-segmentation",
17
- "part-of-speech-tagging",
18
- "language-models",
19
- "ngrams",
20
- "childes",
21
- "talkbank",
22
- "chat",
23
- "averaged-perceptron",
24
- "hidden-markov-model",
25
- "longest-string-matching",
26
15
  ]
27
16
  categories = ["science", "text-processing"]
28
17
 
@@ -32,7 +21,7 @@ name = "rustling"
32
21
  crate-type = ["rlib"]
33
22
 
34
23
  [dependencies]
35
- pyo3 = { version = "0.28.2", features = ["abi3-py310"] }
24
+ pyo3 = { version = "0.28.2", features = ["abi3-py310"], optional = true }
36
25
  rand = "0.10"
37
26
  rayon = { version = "1.10", optional = true }
38
27
  fancy-regex = "0.17"
@@ -51,5 +40,5 @@ tempfile = "3"
51
40
 
52
41
  [features]
53
42
  default = ["parallel"]
54
- extension-module = ["pyo3/extension-module"]
43
+ extension-module = ["pyo3", "pyo3/extension-module"]
55
44
  parallel = ["rayon"]
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rustling
3
- Version: 0.6.0
4
- Classifier: Development Status :: 2 - Pre-Alpha
3
+ Version: 0.7.0
4
+ Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Education
7
7
  Classifier: Intended Audience :: Information Technology
@@ -27,25 +27,27 @@ Classifier: Topic :: Text Processing :: Indexing
27
27
  Classifier: Topic :: Text Processing :: Linguistic
28
28
  License-File: LICENSE.md
29
29
  Summary: A blazingly fast library for computational linguistics
30
- Keywords: computational-linguistics,linguistics,natural-language-processing,nlp,text-processing,word-segmentation,part-of-speech-tagging,language-models,ngrams,childes,talkbank,chat,averaged-perceptron,hidden-markov-model,longest-string-matching
30
+ Keywords: linguistics,nlp,text-processing
31
31
  Author-email: "Jackson L. Lee" <jacksonlunlee@gmail.com>
32
32
  License-Expression: MIT
33
33
  Requires-Python: >=3.10
34
34
  Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
35
- Project-URL: Changelog, https://github.com/jacksonllee/rustling/blob/main/CHANGELOG.md
36
- Project-URL: Python Documentation, https://rustling.readthedocs.io
37
- Project-URL: Rust Documentation, https://docs.rs/rustling
38
35
  Project-URL: Source, https://github.com/jacksonllee/rustling
39
36
 
40
- <img src="https://raw.githubusercontent.com/jacksonllee/rustling/main/python/docs/_static/logo-with-text.svg" alt="Rustling" height="120">
37
+ <div align="center">
38
+ <a href="https://github.com/jacksonllee/rustling">
39
+ <img src="https://raw.githubusercontent.com/jacksonllee/rustling/main/python/docs/_static/logo-with-text.svg" alt="Rustling" height="120">
40
+ </a>
41
+ </div>
42
+ <br>
41
43
 
42
44
  [![PyPI](https://img.shields.io/pypi/v/rustling.svg)](https://pypi.org/project/rustling/)
43
45
  [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
46
+ [![Conda Version](https://img.shields.io/conda/vn/conda-forge/rustling.svg)](https://anaconda.org/conda-forge/rustling)
44
47
 
45
48
  Rustling is a blazingly fast library for computational linguistics.
46
- It is written in Rust, with Python bindings.
47
49
 
48
- Documentation: [Python](https://rustling.readthedocs.io/) | [Rust](https://docs.rs/rustling)
50
+ Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
49
51
 
50
52
  ## Features
51
53
 
@@ -60,19 +62,19 @@ Documentation: [Python](https://rustling.readthedocs.io/) | [Rust](https://docs.
60
62
 
61
63
  | Component | Task | Speedup | vs. |
62
64
  |---|---|---|---|
63
- | **Language Models** | Fit | **10x** | NLTK |
64
- | | Score | **1.9x** | NLTK |
65
- | | Generate | **106--114x** | NLTK |
65
+ | **Language Models** | Fit | **11x** | NLTK |
66
+ | | Score | **2x** | NLTK |
67
+ | | Generate | **86--107x** | NLTK |
66
68
  | **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
67
69
  | **POS Tagging** | Training | **5x** | NLTK |
68
- | | Tagging | **18x** | NLTK |
69
- | **HMM** | Fit | **13x** | hmmlearn |
70
+ | | Tagging | **17x** | NLTK |
71
+ | **HMM** | Fit | **14x** | hmmlearn |
70
72
  | | Predict | **0.9x** | hmmlearn |
71
73
  | | Score | **5x** | hmmlearn |
72
- | **CHAT Parsing** | Reading from a ZIP archive | **43x** | pylangacq |
73
- | | Reading from strings | **70x** | pylangacq |
74
+ | **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
75
+ | | Reading from strings | **35x** | pylangacq |
74
76
  | | Parsing utterances | **15x** | pylangacq |
75
- | | Parsing tokens | **9x** | pylangacq |
77
+ | | Parsing tokens | **8x** | pylangacq |
76
78
 
77
79
  See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
78
80
 
@@ -81,10 +83,22 @@ See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks
81
83
 
82
84
  ### Python
83
85
 
86
+ Using pip:
87
+
84
88
  ```bash
85
89
  pip install rustling
86
90
  ```
87
91
 
92
+ Using conda:
93
+
94
+ ```bash
95
+ conda install -c conda-forge rustling
96
+ ```
97
+
98
+ For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
99
+ are available from each [GitHub release](https://github.com/jacksonllee/rustling/releases)
100
+ — look for the ``.whl`` file with ``emscripten`` in the filename.
101
+
88
102
  ### Rust
89
103
 
90
104
  ```bash
@@ -0,0 +1,74 @@
1
+ <div align="center">
2
+ <a href="https://github.com/jacksonllee/rustling">
3
+ <img src="https://raw.githubusercontent.com/jacksonllee/rustling/main/python/docs/_static/logo-with-text.svg" alt="Rustling" height="120">
4
+ </a>
5
+ </div>
6
+ <br>
7
+
8
+ [![PyPI](https://img.shields.io/pypi/v/rustling.svg)](https://pypi.org/project/rustling/)
9
+ [![crates.io](https://img.shields.io/crates/v/rustling.svg)](https://crates.io/crates/rustling)
10
+ [![Conda Version](https://img.shields.io/conda/vn/conda-forge/rustling.svg)](https://anaconda.org/conda-forge/rustling)
11
+
12
+ Rustling is a blazingly fast library for computational linguistics.
13
+
14
+ Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
15
+
16
+ ## Features
17
+
18
+ - N-grams
19
+ - Language models
20
+ - Hidden Markov model
21
+ - Word segmentation
22
+ - Part-of-speech tagging
23
+ - CHAT parsing for TalkBank and CHILDES data
24
+
25
+ ## Performance
26
+
27
+ | Component | Task | Speedup | vs. |
28
+ |---|---|---|---|
29
+ | **Language Models** | Fit | **11x** | NLTK |
30
+ | | Score | **2x** | NLTK |
31
+ | | Generate | **86--107x** | NLTK |
32
+ | **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
33
+ | **POS Tagging** | Training | **5x** | NLTK |
34
+ | | Tagging | **17x** | NLTK |
35
+ | **HMM** | Fit | **14x** | hmmlearn |
36
+ | | Predict | **0.9x** | hmmlearn |
37
+ | | Score | **5x** | hmmlearn |
38
+ | **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
39
+ | | Reading from strings | **35x** | pylangacq |
40
+ | | Parsing utterances | **15x** | pylangacq |
41
+ | | Parsing tokens | **8x** | pylangacq |
42
+
43
+ See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
44
+
45
+
46
+ ## Installation
47
+
48
+ ### Python
49
+
50
+ Using pip:
51
+
52
+ ```bash
53
+ pip install rustling
54
+ ```
55
+
56
+ Using conda:
57
+
58
+ ```bash
59
+ conda install -c conda-forge rustling
60
+ ```
61
+
62
+ For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
63
+ are available from each [GitHub release](https://github.com/jacksonllee/rustling/releases)
64
+ — look for the ``.whl`` file with ``emscripten`` in the filename.
65
+
66
+ ### Rust
67
+
68
+ ```bash
69
+ cargo add rustling
70
+ ```
71
+
72
+ ## License
73
+
74
+ MIT License
@@ -51,7 +51,7 @@ uv sync --group benchmarks
51
51
 
52
52
  All benchmarks degrade gracefully if a comparison library is not installed.
53
53
 
54
- For benchmark results, see the [performance page](https://rustling.readthedocs.io/#performance).
54
+ For benchmark results, see the [performance page](https://docs.rustling.io/#performance).
55
55
 
56
56
  ---
57
57
 
@@ -10,7 +10,7 @@ requires-python = ">=3.10"
10
10
  license = "MIT"
11
11
  authors = [ { name = "Jackson L. Lee", email = "jacksonlunlee@gmail.com" } ]
12
12
  classifiers = [
13
- "Development Status :: 2 - Pre-Alpha",
13
+ "Development Status :: 3 - Alpha",
14
14
  "Intended Audience :: Developers",
15
15
  "Intended Audience :: Education",
16
16
  "Intended Audience :: Information Technology",
@@ -38,10 +38,7 @@ classifiers = [
38
38
  dynamic = ["version", "keywords"]
39
39
 
40
40
  [project.urls]
41
- "Python Documentation" = "https://rustling.readthedocs.io"
42
- "Rust Documentation" = "https://docs.rs/rustling"
43
41
  Source = "https://github.com/jacksonllee/rustling"
44
- Changelog = "https://github.com/jacksonllee/rustling/blob/main/CHANGELOG.md"
45
42
 
46
43
  [tool.maturin]
47
44
  features = ["extension-module", "parallel"]
@@ -12,7 +12,6 @@ Rustling
12
12
  rust, python
13
13
 
14
14
  Rustling is a blazingly fast library for computational linguistics.
15
- It is written in Rust, with Python bindings.
16
15
 
17
16
  .. toctree::
18
17
  :maxdepth: 1
@@ -29,10 +28,24 @@ It is written in Rust, with Python bindings.
29
28
  Installation
30
29
  ------------
31
30
 
31
+ Using pip:
32
+
32
33
  .. code-block:: bash
33
34
 
34
35
  pip install rustling
35
36
 
37
+ Using conda:
38
+
39
+ .. code-block:: bash
40
+
41
+ conda install -c conda-forge rustling
42
+
43
+ For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
44
+ are available from each `GitHub release <https://github.com/jacksonllee/rustling/releases>`_
45
+ — look for the ``.whl`` file with ``emscripten`` in the filename.
46
+
47
+ Rustling is also available in `Rust <https://docs.rs/rustling>`_.
48
+
36
49
 
37
50
  Performance
38
51
  -----------
@@ -52,15 +65,15 @@ for full details and reproduction scripts.
52
65
  - vs.
53
66
  * - **Language Models**
54
67
  - Fit
55
- - **10x**
68
+ - **11x**
56
69
  - NLTK
57
70
  * -
58
71
  - Score
59
- - **1.9x**
72
+ - **2x**
60
73
  - NLTK
61
74
  * -
62
75
  - Generate
63
- - **106--114x**
76
+ - **86--107x**
64
77
  - NLTK
65
78
  * - **Word Segmentation**
66
79
  - LongestStringMatching
@@ -72,11 +85,11 @@ for full details and reproduction scripts.
72
85
  - NLTK
73
86
  * -
74
87
  - Tagging
75
- - **18x**
88
+ - **17x**
76
89
  - NLTK
77
90
  * - **HMM**
78
91
  - Fit
79
- - **13x**
92
+ - **14x**
80
93
  - hmmlearn
81
94
  * -
82
95
  - Predict
@@ -88,19 +101,19 @@ for full details and reproduction scripts.
88
101
  - hmmlearn
89
102
  * - **CHAT Parsing**
90
103
  - Reading from a ZIP archive
91
- - **43x**
104
+ - **30x**
92
105
  - pylangacq
93
- * -
106
+ * -
94
107
  - Reading from strings
95
- - **70x**
108
+ - **35x**
96
109
  - pylangacq
97
- * -
110
+ * -
98
111
  - Parsing utterances
99
112
  - **15x**
100
113
  - pylangacq
101
- * -
114
+ * -
102
115
  - Parsing tokens
103
- - **9x**
116
+ - **8x**
104
117
  - pylangacq
105
118
 
106
119
 
@@ -20,24 +20,24 @@ with BMES tagging and Viterbi decoding to segment unsegmented text into words.
20
20
  print(result)
21
21
  # [['that', 'is', 'a', 'd', 'o', 'g'], ['this', 'is', 'not', 'a', 'c', 'a', 't']]
22
22
 
23
- Averaged Perceptron
24
- -------------------
23
+ DAG-HMM Segmenter
24
+ ------------------
25
25
 
26
- The :py:class:`~rustling.wordseg.AveragedPerceptronSegmenter` uses a character-level averaged perceptron
27
- with BMES tagging to segment unsegmented text into words.
26
+ The :py:class:`~rustling.wordseg.DAGHMMSegmenter` is a jieba-style hybrid segmenter that combines
27
+ dictionary-based DAG (directed acyclic graph) segmentation with an HMM fallback for
28
+ out-of-vocabulary spans.
28
29
 
29
30
  .. code-block:: python
30
31
 
31
- from rustling.wordseg import AveragedPerceptronSegmenter
32
+ from rustling.wordseg import DAGHMMSegmenter
32
33
 
33
- model = AveragedPerceptronSegmenter(n_iter=5)
34
- model.fit([
34
+ model = DAGHMMSegmenter()
35
+ model.fit_segmented([
35
36
  ("this", "is", "a", "sentence"),
36
37
  ("that", "is", "not", "a", "sentence"),
37
38
  ])
38
39
  result = model.predict(["thatisadog", "thisisnotacat"])
39
40
  print(result)
40
- # [['that', 'is', 'a', 'dog'], ['this', 'is', 'not', 'a', 'cat']]
41
41
 
42
42
  Longest String Matching
43
43
  ------------------------