rustling 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rustling-0.7.0/.github/workflows/build-wheel.yml +42 -0
- {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/python.yml +1 -1
- {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/release.yml +2 -1
- {rustling-0.6.0 → rustling-0.7.0}/.pre-commit-config.yaml +11 -1
- rustling-0.7.0/CHANGELOG.md +70 -0
- {rustling-0.6.0 → rustling-0.7.0}/Cargo.lock +11 -11
- {rustling-0.6.0 → rustling-0.7.0}/Cargo.toml +4 -15
- {rustling-0.6.0 → rustling-0.7.0}/PKG-INFO +31 -17
- rustling-0.7.0/README.md +74 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/README.md +1 -1
- {rustling-0.6.0 → rustling-0.7.0}/pyproject.toml +1 -4
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/index.rst +25 -12
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/wordseg.rst +8 -8
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/chat/__init__.pyi +49 -4
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/wordseg/__init__.pyi +113 -9
- {rustling-0.6.0 → rustling-0.7.0}/python/stubtest_allowlist.txt +4 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_chat/test_chat.py +146 -14
- rustling-0.7.0/python/tests/test_wordseg/test_dag_hmm_segmenter.py +69 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_hmm_segmenter.py +42 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_longest_string_matching.py +36 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_wordseg/test_random_segmenter.py +31 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/clean_utterance.rs +481 -11
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/header.rs +85 -140
- rustling-0.7.0/src/chat/header_py.rs +312 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/ipsyn.rs +4 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/mod.rs +16 -5
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/reader.rs +307 -1128
- rustling-0.7.0/src/chat/reader_py.rs +1128 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/chat/utterance.rs +108 -309
- rustling-0.7.0/src/chat/utterance_py.rs +330 -0
- rustling-0.7.0/src/chat/validation.rs +1091 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/hmm/mod.rs +52 -276
- rustling-0.7.0/src/hmm/py.rs +254 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/lib.rs +4 -1
- {rustling-0.6.0 → rustling-0.7.0}/src/lm/mod.rs +40 -382
- rustling-0.7.0/src/lm/py.rs +370 -0
- rustling-0.6.0/src/ngram.rs → rustling-0.7.0/src/ngram/mod.rs +30 -222
- rustling-0.7.0/src/ngram/py.rs +204 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/mod.rs +8 -141
- rustling-0.7.0/src/perceptron_pos_tagger/py.rs +144 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/persistence.rs +28 -2
- rustling-0.7.0/src/prelude.rs +15 -0
- rustling-0.6.0/src/seq_feature.rs → rustling-0.7.0/src/seq_feature/mod.rs +23 -80
- rustling-0.7.0/src/seq_feature/py.rs +74 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/dag_hmm.rs +45 -60
- rustling-0.7.0/src/wordseg/dag_hmm_py.rs +76 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/hmm.rs +86 -83
- rustling-0.7.0/src/wordseg/hmm_py.rs +99 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/mod.rs +15 -100
- rustling-0.7.0/src/wordseg/longest_string_matching/py.rs +109 -0
- rustling-0.7.0/src/wordseg/mod.rs +141 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/random_segmenter.rs +12 -59
- rustling-0.7.0/src/wordseg/random_segmenter_py.rs +68 -0
- rustling-0.6.0/AGENTS.md +0 -335
- rustling-0.6.0/CHANGELOG.md +0 -42
- rustling-0.6.0/CLAUDE.md +0 -335
- rustling-0.6.0/README.md +0 -57
- rustling-0.6.0/src/wordseg/mod.rs +0 -55
- {rustling-0.6.0 → rustling-0.7.0}/.flake8 +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/.gitattributes +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/.github/dependabot.yml +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/.github/workflows/rust.yml +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/.gitignore +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/.readthedocs.yaml +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/LICENSE.md +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/common/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/common/data.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_chat.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_hmm.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_lm.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_perceptron_pos_tagger.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/run_wordseg.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/benchmarks/update_docs.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/build.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/Makefile +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/README.md +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/_static/favicon.ico +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/_static/logo-with-text.svg +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/_templates/copyright.html +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/api.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/chat.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/conf.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/hmm.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/lm.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/make.bat +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/ngram.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/perceptron_pos_tagger.rst +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/docs/requirements.txt +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/chat/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/hmm/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/hmm/__init__.pyi +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/lm/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/lm/__init__.pyi +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/ngram/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/ngram/__init__.pyi +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/perceptron_pos_tagger/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/perceptron_pos_tagger/__init__.pyi +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/py.typed +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/seq_feature/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/seq_feature/__init__.pyi +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/rustling/wordseg/__init__.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_benchmarks.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_chat/conftest.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_hmm.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_lm.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_ngram.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/python/tests/test_perceptron_pos_tagger.py +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/hmm/model.fbs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/hmm/model_generated.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/lm/model.fbs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/lm/model_generated.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/model.fbs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/perceptron_pos_tagger/model_generated.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/trie.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/bmes.rs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/model.fbs +0 -0
- {rustling-0.6.0 → rustling-0.7.0}/src/wordseg/longest_string_matching/model_generated.rs +0 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: Build Linux x86_64 wheel
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
inputs:
|
|
6
|
+
git_ref:
|
|
7
|
+
description: "Git ref to build from (tag, branch, or commit)"
|
|
8
|
+
required: true
|
|
9
|
+
default: "v0.6.0"
|
|
10
|
+
|
|
11
|
+
env:
|
|
12
|
+
FLATC_VERSION: "25.12.19"
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
build:
|
|
16
|
+
name: Build manylinux x86_64 wheel
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v6
|
|
20
|
+
with:
|
|
21
|
+
ref: ${{ github.event.inputs.git_ref }}
|
|
22
|
+
|
|
23
|
+
- name: Build wheels
|
|
24
|
+
uses: PyO3/maturin-action@v1
|
|
25
|
+
with:
|
|
26
|
+
target: x86_64
|
|
27
|
+
args: --release --out dist -i python3.10
|
|
28
|
+
sccache: "true"
|
|
29
|
+
manylinux: auto
|
|
30
|
+
before-script-linux: |
|
|
31
|
+
curl -sL "https://github.com/google/flatbuffers/releases/download/v${{ env.FLATC_VERSION }}/Linux.flatc.binary.clang++-18.zip" -o flatc.zip
|
|
32
|
+
command -v unzip || yum install -y unzip 2>/dev/null || (apt-get update -qq && apt-get install -y -qq unzip)
|
|
33
|
+
unzip -q flatc.zip
|
|
34
|
+
rm flatc.zip
|
|
35
|
+
chmod +x flatc
|
|
36
|
+
mv flatc /usr/local/bin/
|
|
37
|
+
|
|
38
|
+
- name: Upload wheel
|
|
39
|
+
uses: actions/upload-artifact@v7
|
|
40
|
+
with:
|
|
41
|
+
name: wheel-linux-x86_64
|
|
42
|
+
path: dist
|
|
@@ -71,7 +71,7 @@ jobs:
|
|
|
71
71
|
uses: PyO3/maturin-action@v1
|
|
72
72
|
with:
|
|
73
73
|
target: ${{ matrix.target }}
|
|
74
|
-
args: --release --out dist
|
|
74
|
+
args: --release --out dist -i python3.10
|
|
75
75
|
sccache: "true"
|
|
76
76
|
manylinux: auto
|
|
77
77
|
before-script-linux: |
|
|
@@ -201,6 +201,7 @@ jobs:
|
|
|
201
201
|
run: |
|
|
202
202
|
curl -sL "https://github.com/google/flatbuffers/releases/download/v${{ env.FLATC_VERSION }}/Linux.flatc.binary.clang++-18.zip" -o flatc.zip
|
|
203
203
|
unzip -q flatc.zip
|
|
204
|
+
rm flatc.zip
|
|
204
205
|
chmod +x flatc
|
|
205
206
|
sudo mv flatc /usr/local/bin/
|
|
206
207
|
|
|
@@ -14,7 +14,7 @@ repos:
|
|
|
14
14
|
types: [rust]
|
|
15
15
|
pass_filenames: false
|
|
16
16
|
- repo: https://github.com/psf/black-pre-commit-mirror
|
|
17
|
-
rev: 26.
|
|
17
|
+
rev: 26.3.0
|
|
18
18
|
hooks:
|
|
19
19
|
- id: black
|
|
20
20
|
files: ^(python|benchmarks)/
|
|
@@ -23,3 +23,13 @@ repos:
|
|
|
23
23
|
hooks:
|
|
24
24
|
- id: flake8
|
|
25
25
|
files: ^(python|benchmarks)/
|
|
26
|
+
- repo: local
|
|
27
|
+
hooks:
|
|
28
|
+
- id: mypy
|
|
29
|
+
name: mypy
|
|
30
|
+
entry: uvx mypy
|
|
31
|
+
language: system
|
|
32
|
+
types: [python]
|
|
33
|
+
files: ^python/rustling/
|
|
34
|
+
pass_filenames: false
|
|
35
|
+
args: [python/rustling/]
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.7.0] - 2026-03-14
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Word segmentation:
|
|
15
|
+
* Added `score` method for the HMM and DAG-HMM segmenters.
|
|
16
|
+
* `predict` method can optionally output offsets for the (start, end) indices
|
|
17
|
+
of segmented words compared to the original string.
|
|
18
|
+
- CHAT parsing: Support custom tier names other than the standard %mor and %gra.
|
|
19
|
+
- Python model classes are now subclassable.
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- Ngram counters: `Ngrams.most_common` now sorts tuples lexicographically
|
|
24
|
+
when counts are tied.
|
|
25
|
+
- CHAT parsing:
|
|
26
|
+
* If a date is available at `Headers`'s `date`,
|
|
27
|
+
it's now a Python `datetime.date` object instead of a string.
|
|
28
|
+
* In handling the main tier transcription for creating `Token` objects:
|
|
29
|
+
- Special form markers suffixed with "@" are now stripped.
|
|
30
|
+
- Words that have partiallly parenthetical material have the parentheses
|
|
31
|
+
removed, e.g., (un)til -> until, sit(ting) -> sitting.
|
|
32
|
+
* Renamed the `CHAT.raw` attribute to `CHAT.audible` for a best-effort,
|
|
33
|
+
audibly faithful transcription string, to facilitate automatic speech recognition,
|
|
34
|
+
forced alignment, etc.
|
|
35
|
+
* A subset of the testchat/bad dataset is now used to validate CHAT data format.
|
|
36
|
+
- Refactored core Rust code so that Rust-only consumers no longer need PyO3/Python.
|
|
37
|
+
|
|
38
|
+
## [0.6.0] - 2026-03-05
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
|
|
42
|
+
- Hidden Markov Model (HMM)
|
|
43
|
+
- Word segmentation: Added DAG-HMM word segmenter
|
|
44
|
+
- CHAT parsing: Added `from_utterances` method
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
|
|
48
|
+
- Models are now persisted as a zstd-compressed FlatBuffers binary.
|
|
49
|
+
|
|
50
|
+
## [0.5.0] - 2026-02-18
|
|
51
|
+
|
|
52
|
+
### Added
|
|
53
|
+
|
|
54
|
+
- CHAT parsing for TalkBank and CHILDES data
|
|
55
|
+
|
|
56
|
+
## [0.4.0] - 2026-02-08
|
|
57
|
+
|
|
58
|
+
### Added
|
|
59
|
+
|
|
60
|
+
- N-grams and language models
|
|
61
|
+
|
|
62
|
+
## [0.3.0] - 2026-02-06
|
|
63
|
+
|
|
64
|
+
### Added
|
|
65
|
+
|
|
66
|
+
- Averaged perceptron tagger
|
|
67
|
+
|
|
68
|
+
## [0.2.0] - 2026-02-04
|
|
69
|
+
|
|
70
|
+
- Initial release, with longest string matching and random segmenter for word segmentation
|
|
@@ -81,9 +81,9 @@ dependencies = [
|
|
|
81
81
|
|
|
82
82
|
[[package]]
|
|
83
83
|
name = "cc"
|
|
84
|
-
version = "1.2.
|
|
84
|
+
version = "1.2.57"
|
|
85
85
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
86
|
-
checksum = "
|
|
86
|
+
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
|
|
87
87
|
dependencies = [
|
|
88
88
|
"find-msvc-tools",
|
|
89
89
|
"jobserver",
|
|
@@ -418,9 +418,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7"
|
|
|
418
418
|
|
|
419
419
|
[[package]]
|
|
420
420
|
name = "libc"
|
|
421
|
-
version = "0.2.
|
|
421
|
+
version = "0.2.183"
|
|
422
422
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
423
|
-
checksum = "
|
|
423
|
+
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
|
|
424
424
|
|
|
425
425
|
[[package]]
|
|
426
426
|
name = "linux-raw-sys"
|
|
@@ -467,9 +467,9 @@ checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
|
|
|
467
467
|
|
|
468
468
|
[[package]]
|
|
469
469
|
name = "once_cell"
|
|
470
|
-
version = "1.21.
|
|
470
|
+
version = "1.21.4"
|
|
471
471
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
472
|
-
checksum = "
|
|
472
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
473
473
|
|
|
474
474
|
[[package]]
|
|
475
475
|
name = "pbkdf2"
|
|
@@ -699,7 +699,7 @@ dependencies = [
|
|
|
699
699
|
|
|
700
700
|
[[package]]
|
|
701
701
|
name = "rustling"
|
|
702
|
-
version = "0.
|
|
702
|
+
version = "0.7.0"
|
|
703
703
|
dependencies = [
|
|
704
704
|
"fancy-regex",
|
|
705
705
|
"flatbuffers",
|
|
@@ -840,9 +840,9 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
|
840
840
|
|
|
841
841
|
[[package]]
|
|
842
842
|
name = "tempfile"
|
|
843
|
-
version = "3.
|
|
843
|
+
version = "3.27.0"
|
|
844
844
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
845
|
-
checksum = "
|
|
845
|
+
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
|
846
846
|
dependencies = [
|
|
847
847
|
"fastrand",
|
|
848
848
|
"getrandom 0.4.2",
|
|
@@ -897,9 +897,9 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
|
|
897
897
|
|
|
898
898
|
[[package]]
|
|
899
899
|
name = "uuid"
|
|
900
|
-
version = "1.
|
|
900
|
+
version = "1.22.0"
|
|
901
901
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
902
|
-
checksum = "
|
|
902
|
+
checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
|
|
903
903
|
dependencies = [
|
|
904
904
|
"getrandom 0.4.2",
|
|
905
905
|
"js-sys",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "rustling"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.7.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
description = "A blazingly fast library for computational linguistics"
|
|
6
6
|
license = "MIT"
|
|
@@ -8,21 +8,10 @@ repository = "https://github.com/jacksonllee/rustling"
|
|
|
8
8
|
documentation = "https://docs.rs/rustling"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
keywords = [
|
|
11
|
-
|
|
11
|
+
# At most 5 keywords per crate. Each keyword must have fewer than 20 characters.
|
|
12
12
|
"linguistics",
|
|
13
|
-
"natural-language-processing",
|
|
14
13
|
"nlp",
|
|
15
14
|
"text-processing",
|
|
16
|
-
"word-segmentation",
|
|
17
|
-
"part-of-speech-tagging",
|
|
18
|
-
"language-models",
|
|
19
|
-
"ngrams",
|
|
20
|
-
"childes",
|
|
21
|
-
"talkbank",
|
|
22
|
-
"chat",
|
|
23
|
-
"averaged-perceptron",
|
|
24
|
-
"hidden-markov-model",
|
|
25
|
-
"longest-string-matching",
|
|
26
15
|
]
|
|
27
16
|
categories = ["science", "text-processing"]
|
|
28
17
|
|
|
@@ -32,7 +21,7 @@ name = "rustling"
|
|
|
32
21
|
crate-type = ["rlib"]
|
|
33
22
|
|
|
34
23
|
[dependencies]
|
|
35
|
-
pyo3 = { version = "0.28.2", features = ["abi3-py310"] }
|
|
24
|
+
pyo3 = { version = "0.28.2", features = ["abi3-py310"], optional = true }
|
|
36
25
|
rand = "0.10"
|
|
37
26
|
rayon = { version = "1.10", optional = true }
|
|
38
27
|
fancy-regex = "0.17"
|
|
@@ -51,5 +40,5 @@ tempfile = "3"
|
|
|
51
40
|
|
|
52
41
|
[features]
|
|
53
42
|
default = ["parallel"]
|
|
54
|
-
extension-module = ["pyo3/extension-module"]
|
|
43
|
+
extension-module = ["pyo3", "pyo3/extension-module"]
|
|
55
44
|
parallel = ["rayon"]
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rustling
|
|
3
|
-
Version: 0.
|
|
4
|
-
Classifier: Development Status ::
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Education
|
|
7
7
|
Classifier: Intended Audience :: Information Technology
|
|
@@ -27,25 +27,27 @@ Classifier: Topic :: Text Processing :: Indexing
|
|
|
27
27
|
Classifier: Topic :: Text Processing :: Linguistic
|
|
28
28
|
License-File: LICENSE.md
|
|
29
29
|
Summary: A blazingly fast library for computational linguistics
|
|
30
|
-
Keywords:
|
|
30
|
+
Keywords: linguistics,nlp,text-processing
|
|
31
31
|
Author-email: "Jackson L. Lee" <jacksonlunlee@gmail.com>
|
|
32
32
|
License-Expression: MIT
|
|
33
33
|
Requires-Python: >=3.10
|
|
34
34
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
35
|
-
Project-URL: Changelog, https://github.com/jacksonllee/rustling/blob/main/CHANGELOG.md
|
|
36
|
-
Project-URL: Python Documentation, https://rustling.readthedocs.io
|
|
37
|
-
Project-URL: Rust Documentation, https://docs.rs/rustling
|
|
38
35
|
Project-URL: Source, https://github.com/jacksonllee/rustling
|
|
39
36
|
|
|
40
|
-
<
|
|
37
|
+
<div align="center">
|
|
38
|
+
<a href="https://github.com/jacksonllee/rustling">
|
|
39
|
+
<img src="https://raw.githubusercontent.com/jacksonllee/rustling/main/python/docs/_static/logo-with-text.svg" alt="Rustling" height="120">
|
|
40
|
+
</a>
|
|
41
|
+
</div>
|
|
42
|
+
<br>
|
|
41
43
|
|
|
42
44
|
[](https://pypi.org/project/rustling/)
|
|
43
45
|
[](https://crates.io/crates/rustling)
|
|
46
|
+
[](https://anaconda.org/conda-forge/rustling)
|
|
44
47
|
|
|
45
48
|
Rustling is a blazingly fast library for computational linguistics.
|
|
46
|
-
It is written in Rust, with Python bindings.
|
|
47
49
|
|
|
48
|
-
Documentation: [Python](https://rustling.
|
|
50
|
+
Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
|
|
49
51
|
|
|
50
52
|
## Features
|
|
51
53
|
|
|
@@ -60,19 +62,19 @@ Documentation: [Python](https://rustling.readthedocs.io/) | [Rust](https://docs.
|
|
|
60
62
|
|
|
61
63
|
| Component | Task | Speedup | vs. |
|
|
62
64
|
|---|---|---|---|
|
|
63
|
-
| **Language Models** | Fit | **
|
|
64
|
-
| | Score | **
|
|
65
|
-
| | Generate | **
|
|
65
|
+
| **Language Models** | Fit | **11x** | NLTK |
|
|
66
|
+
| | Score | **2x** | NLTK |
|
|
67
|
+
| | Generate | **86--107x** | NLTK |
|
|
66
68
|
| **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
|
|
67
69
|
| **POS Tagging** | Training | **5x** | NLTK |
|
|
68
|
-
| | Tagging | **
|
|
69
|
-
| **HMM** | Fit | **
|
|
70
|
+
| | Tagging | **17x** | NLTK |
|
|
71
|
+
| **HMM** | Fit | **14x** | hmmlearn |
|
|
70
72
|
| | Predict | **0.9x** | hmmlearn |
|
|
71
73
|
| | Score | **5x** | hmmlearn |
|
|
72
|
-
| **CHAT Parsing** | Reading from a ZIP archive | **
|
|
73
|
-
| | Reading from strings | **
|
|
74
|
+
| **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
|
|
75
|
+
| | Reading from strings | **35x** | pylangacq |
|
|
74
76
|
| | Parsing utterances | **15x** | pylangacq |
|
|
75
|
-
| | Parsing tokens | **
|
|
77
|
+
| | Parsing tokens | **8x** | pylangacq |
|
|
76
78
|
|
|
77
79
|
See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
|
|
78
80
|
|
|
@@ -81,10 +83,22 @@ See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks
|
|
|
81
83
|
|
|
82
84
|
### Python
|
|
83
85
|
|
|
86
|
+
Using pip:
|
|
87
|
+
|
|
84
88
|
```bash
|
|
85
89
|
pip install rustling
|
|
86
90
|
```
|
|
87
91
|
|
|
92
|
+
Using conda:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
conda install -c conda-forge rustling
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
|
|
99
|
+
are available from each [GitHub release](https://github.com/jacksonllee/rustling/releases)
|
|
100
|
+
— look for the ``.whl`` file with ``emscripten`` in the filename.
|
|
101
|
+
|
|
88
102
|
### Rust
|
|
89
103
|
|
|
90
104
|
```bash
|
rustling-0.7.0/README.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
<a href="https://github.com/jacksonllee/rustling">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/jacksonllee/rustling/main/python/docs/_static/logo-with-text.svg" alt="Rustling" height="120">
|
|
4
|
+
</a>
|
|
5
|
+
</div>
|
|
6
|
+
<br>
|
|
7
|
+
|
|
8
|
+
[](https://pypi.org/project/rustling/)
|
|
9
|
+
[](https://crates.io/crates/rustling)
|
|
10
|
+
[](https://anaconda.org/conda-forge/rustling)
|
|
11
|
+
|
|
12
|
+
Rustling is a blazingly fast library for computational linguistics.
|
|
13
|
+
|
|
14
|
+
Documentation: [Python](https://docs.rustling.io/) | [Rust](https://docs.rs/rustling)
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
- N-grams
|
|
19
|
+
- Language models
|
|
20
|
+
- Hidden Markov model
|
|
21
|
+
- Word segmentation
|
|
22
|
+
- Part-of-speech tagging
|
|
23
|
+
- CHAT parsing for TalkBank and CHILDES data
|
|
24
|
+
|
|
25
|
+
## Performance
|
|
26
|
+
|
|
27
|
+
| Component | Task | Speedup | vs. |
|
|
28
|
+
|---|---|---|---|
|
|
29
|
+
| **Language Models** | Fit | **11x** | NLTK |
|
|
30
|
+
| | Score | **2x** | NLTK |
|
|
31
|
+
| | Generate | **86--107x** | NLTK |
|
|
32
|
+
| **Word Segmentation** | LongestStringMatching | **9x** | wordseg |
|
|
33
|
+
| **POS Tagging** | Training | **5x** | NLTK |
|
|
34
|
+
| | Tagging | **17x** | NLTK |
|
|
35
|
+
| **HMM** | Fit | **14x** | hmmlearn |
|
|
36
|
+
| | Predict | **0.9x** | hmmlearn |
|
|
37
|
+
| | Score | **5x** | hmmlearn |
|
|
38
|
+
| **CHAT Parsing** | Reading from a ZIP archive | **30x** | pylangacq |
|
|
39
|
+
| | Reading from strings | **35x** | pylangacq |
|
|
40
|
+
| | Parsing utterances | **15x** | pylangacq |
|
|
41
|
+
| | Parsing tokens | **8x** | pylangacq |
|
|
42
|
+
|
|
43
|
+
See [`benchmarks/`](https://github.com/jacksonllee/rustling/tree/main/benchmarks) for reproduction scripts.
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
### Python
|
|
49
|
+
|
|
50
|
+
Using pip:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install rustling
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Using conda:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
conda install -c conda-forge rustling
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
|
|
63
|
+
are available from each [GitHub release](https://github.com/jacksonllee/rustling/releases)
|
|
64
|
+
— look for the ``.whl`` file with ``emscripten`` in the filename.
|
|
65
|
+
|
|
66
|
+
### Rust
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
cargo add rustling
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## License
|
|
73
|
+
|
|
74
|
+
MIT License
|
|
@@ -51,7 +51,7 @@ uv sync --group benchmarks
|
|
|
51
51
|
|
|
52
52
|
All benchmarks degrade gracefully if a comparison library is not installed.
|
|
53
53
|
|
|
54
|
-
For benchmark results, see the [performance page](https://rustling.
|
|
54
|
+
For benchmark results, see the [performance page](https://docs.rustling.io/#performance).
|
|
55
55
|
|
|
56
56
|
---
|
|
57
57
|
|
|
@@ -10,7 +10,7 @@ requires-python = ">=3.10"
|
|
|
10
10
|
license = "MIT"
|
|
11
11
|
authors = [ { name = "Jackson L. Lee", email = "jacksonlunlee@gmail.com" } ]
|
|
12
12
|
classifiers = [
|
|
13
|
-
"Development Status ::
|
|
13
|
+
"Development Status :: 3 - Alpha",
|
|
14
14
|
"Intended Audience :: Developers",
|
|
15
15
|
"Intended Audience :: Education",
|
|
16
16
|
"Intended Audience :: Information Technology",
|
|
@@ -38,10 +38,7 @@ classifiers = [
|
|
|
38
38
|
dynamic = ["version", "keywords"]
|
|
39
39
|
|
|
40
40
|
[project.urls]
|
|
41
|
-
"Python Documentation" = "https://rustling.readthedocs.io"
|
|
42
|
-
"Rust Documentation" = "https://docs.rs/rustling"
|
|
43
41
|
Source = "https://github.com/jacksonllee/rustling"
|
|
44
|
-
Changelog = "https://github.com/jacksonllee/rustling/blob/main/CHANGELOG.md"
|
|
45
42
|
|
|
46
43
|
[tool.maturin]
|
|
47
44
|
features = ["extension-module", "parallel"]
|
|
@@ -12,7 +12,6 @@ Rustling
|
|
|
12
12
|
rust, python
|
|
13
13
|
|
|
14
14
|
Rustling is a blazingly fast library for computational linguistics.
|
|
15
|
-
It is written in Rust, with Python bindings.
|
|
16
15
|
|
|
17
16
|
.. toctree::
|
|
18
17
|
:maxdepth: 1
|
|
@@ -29,10 +28,24 @@ It is written in Rust, with Python bindings.
|
|
|
29
28
|
Installation
|
|
30
29
|
------------
|
|
31
30
|
|
|
31
|
+
Using pip:
|
|
32
|
+
|
|
32
33
|
.. code-block:: bash
|
|
33
34
|
|
|
34
35
|
pip install rustling
|
|
35
36
|
|
|
37
|
+
Using conda:
|
|
38
|
+
|
|
39
|
+
.. code-block:: bash
|
|
40
|
+
|
|
41
|
+
conda install -c conda-forge rustling
|
|
42
|
+
|
|
43
|
+
For Pyodide, pre-built WASM wheels (with multithreading disabled, as Pyodide does not support it)
|
|
44
|
+
are available from each `GitHub release <https://github.com/jacksonllee/rustling/releases>`_
|
|
45
|
+
— look for the ``.whl`` file with ``emscripten`` in the filename.
|
|
46
|
+
|
|
47
|
+
Rustling is also available in `Rust <https://docs.rs/rustling>`_.
|
|
48
|
+
|
|
36
49
|
|
|
37
50
|
Performance
|
|
38
51
|
-----------
|
|
@@ -52,15 +65,15 @@ for full details and reproduction scripts.
|
|
|
52
65
|
- vs.
|
|
53
66
|
* - **Language Models**
|
|
54
67
|
- Fit
|
|
55
|
-
- **
|
|
68
|
+
- **11x**
|
|
56
69
|
- NLTK
|
|
57
70
|
* -
|
|
58
71
|
- Score
|
|
59
|
-
- **
|
|
72
|
+
- **2x**
|
|
60
73
|
- NLTK
|
|
61
74
|
* -
|
|
62
75
|
- Generate
|
|
63
|
-
- **
|
|
76
|
+
- **86--107x**
|
|
64
77
|
- NLTK
|
|
65
78
|
* - **Word Segmentation**
|
|
66
79
|
- LongestStringMatching
|
|
@@ -72,11 +85,11 @@ for full details and reproduction scripts.
|
|
|
72
85
|
- NLTK
|
|
73
86
|
* -
|
|
74
87
|
- Tagging
|
|
75
|
-
- **
|
|
88
|
+
- **17x**
|
|
76
89
|
- NLTK
|
|
77
90
|
* - **HMM**
|
|
78
91
|
- Fit
|
|
79
|
-
- **
|
|
92
|
+
- **14x**
|
|
80
93
|
- hmmlearn
|
|
81
94
|
* -
|
|
82
95
|
- Predict
|
|
@@ -88,19 +101,19 @@ for full details and reproduction scripts.
|
|
|
88
101
|
- hmmlearn
|
|
89
102
|
* - **CHAT Parsing**
|
|
90
103
|
- Reading from a ZIP archive
|
|
91
|
-
- **
|
|
104
|
+
- **30x**
|
|
92
105
|
- pylangacq
|
|
93
|
-
* -
|
|
106
|
+
* -
|
|
94
107
|
- Reading from strings
|
|
95
|
-
- **
|
|
108
|
+
- **35x**
|
|
96
109
|
- pylangacq
|
|
97
|
-
* -
|
|
110
|
+
* -
|
|
98
111
|
- Parsing utterances
|
|
99
112
|
- **15x**
|
|
100
113
|
- pylangacq
|
|
101
|
-
* -
|
|
114
|
+
* -
|
|
102
115
|
- Parsing tokens
|
|
103
|
-
- **
|
|
116
|
+
- **8x**
|
|
104
117
|
- pylangacq
|
|
105
118
|
|
|
106
119
|
|
|
@@ -20,24 +20,24 @@ with BMES tagging and Viterbi decoding to segment unsegmented text into words.
|
|
|
20
20
|
print(result)
|
|
21
21
|
# [['that', 'is', 'a', 'd', 'o', 'g'], ['this', 'is', 'not', 'a', 'c', 'a', 't']]
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
|
|
23
|
+
DAG-HMM Segmenter
|
|
24
|
+
------------------
|
|
25
25
|
|
|
26
|
-
The :py:class:`~rustling.wordseg.
|
|
27
|
-
|
|
26
|
+
The :py:class:`~rustling.wordseg.DAGHMMSegmenter` is a jieba-style hybrid segmenter that combines
|
|
27
|
+
dictionary-based DAG (directed acyclic graph) segmentation with an HMM fallback for
|
|
28
|
+
out-of-vocabulary spans.
|
|
28
29
|
|
|
29
30
|
.. code-block:: python
|
|
30
31
|
|
|
31
|
-
from rustling.wordseg import
|
|
32
|
+
from rustling.wordseg import DAGHMMSegmenter
|
|
32
33
|
|
|
33
|
-
model =
|
|
34
|
-
model.
|
|
34
|
+
model = DAGHMMSegmenter()
|
|
35
|
+
model.fit_segmented([
|
|
35
36
|
("this", "is", "a", "sentence"),
|
|
36
37
|
("that", "is", "not", "a", "sentence"),
|
|
37
38
|
])
|
|
38
39
|
result = model.predict(["thatisadog", "thisisnotacat"])
|
|
39
40
|
print(result)
|
|
40
|
-
# [['that', 'is', 'a', 'dog'], ['this', 'is', 'not', 'a', 'cat']]
|
|
41
41
|
|
|
42
42
|
Longest String Matching
|
|
43
43
|
------------------------
|