skimtoken 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {skimtoken-0.1.2 → skimtoken-0.2.0}/.gitignore +9 -1
- skimtoken-0.2.0/CONTRIBUTING.md +96 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/Cargo.lock +299 -2
- {skimtoken-0.1.2 → skimtoken-0.2.0}/Cargo.toml +4 -1
- skimtoken-0.2.0/PKG-INFO +281 -0
- skimtoken-0.2.0/README.md +268 -0
- skimtoken-0.2.0/examples/example.py +6 -0
- skimtoken-0.2.0/examples/multilingual_estimate.py +27 -0
- skimtoken-0.2.0/params/basic.toml +5 -0
- skimtoken-0.2.0/params/multilingual.toml +489 -0
- skimtoken-0.2.0/params/multilingual_simple.toml +209 -0
- skimtoken-0.2.0/params/simple.toml +1 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/pyproject.toml +10 -3
- skimtoken-0.2.0/scripts/benchmark.py +401 -0
- skimtoken-0.2.0/scripts/optimize/optimize_basic.py +130 -0
- skimtoken-0.2.0/scripts/optimize/optimize_multilingual.py +352 -0
- skimtoken-0.2.0/scripts/optimize/optimize_multilingual_simple.py +255 -0
- skimtoken-0.2.0/scripts/optimize/optimize_simple.py +128 -0
- skimtoken-0.2.0/scripts/optimize/utils.py +81 -0
- skimtoken-0.2.0/scripts/optimize_all.py +120 -0
- skimtoken-0.2.0/scripts/prepare_cc100_dataset.py +341 -0
- skimtoken-0.2.0/scripts/update_rust_params.py +299 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/scripts/update_token_counts.py +31 -8
- skimtoken-0.2.0/skimtoken/__init__.py +103 -0
- skimtoken-0.2.0/skimtoken/__init__.pyi +58 -0
- skimtoken-0.2.0/skimtoken/basic.py +6 -0
- skimtoken-0.2.0/skimtoken/basic.pyi +9 -0
- skimtoken-0.2.0/skimtoken/multilingual.py +6 -0
- skimtoken-0.2.0/skimtoken/multilingual.pyi +9 -0
- skimtoken-0.2.0/skimtoken/multilingual_simple.py +6 -0
- skimtoken-0.2.0/skimtoken/multilingual_simple.pyi +9 -0
- skimtoken-0.2.0/skimtoken/simple.py +6 -0
- skimtoken-0.2.0/skimtoken/simple.pyi +9 -0
- skimtoken-0.2.0/src/lib.rs +234 -0
- skimtoken-0.2.0/src/main.rs +43 -0
- skimtoken-0.2.0/src/methods/method.rs +41 -0
- skimtoken-0.2.0/src/methods/method_basic.rs +94 -0
- skimtoken-0.2.0/src/methods/method_multilingual.rs +965 -0
- skimtoken-0.2.0/src/methods/method_multilingual_simple.rs +656 -0
- skimtoken-0.2.0/src/methods/method_simple.rs +55 -0
- skimtoken-0.2.0/tests/test_comprehensive.py +323 -0
- skimtoken-0.2.0/tests/test_hypothesis.py +115 -0
- skimtoken-0.1.2/tests/test_skimtoken_simple.py → skimtoken-0.2.0/tests/test_simple.py +4 -22
- skimtoken-0.2.0/uv.lock +2277 -0
- skimtoken-0.1.2/PKG-INFO +0 -185
- skimtoken-0.1.2/README.md +0 -171
- skimtoken-0.1.2/examples/example.py +0 -12
- skimtoken-0.1.2/scripts/benchmark.py +0 -241
- skimtoken-0.1.2/scripts/run_benchmark_multiple.sh +0 -211
- skimtoken-0.1.2/src/lib.rs +0 -649
- skimtoken-0.1.2/src/main.rs +0 -57
- skimtoken-0.1.2/src/skimtoken/__init__.py +0 -6
- skimtoken-0.1.2/src/skimtoken/__init__.pyi +0 -6
- skimtoken-0.1.2/src/skimtoken/cli.py +0 -44
- skimtoken-0.1.2/uv.lock +0 -521
- {skimtoken-0.1.2 → skimtoken-0.2.0}/.github/workflows/ci.yml +0 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/.github/workflows/release.yml +0 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/LICENSE +0 -0
- {skimtoken-0.1.2 → skimtoken-0.2.0}/data/test_dataset.jsonl +0 -0
|
@@ -15,6 +15,12 @@ env/
|
|
|
15
15
|
coverage.*
|
|
16
16
|
wheels/
|
|
17
17
|
|
|
18
|
+
# Cache
|
|
19
|
+
.hypothesis/
|
|
20
|
+
.pytest_cache/
|
|
21
|
+
.ruff_cache/
|
|
22
|
+
|
|
23
|
+
|
|
18
24
|
# Rust
|
|
19
25
|
/target/
|
|
20
26
|
Cargo.lock
|
|
@@ -34,4 +40,6 @@ Thumbs.db
|
|
|
34
40
|
# Project specific
|
|
35
41
|
*.dylib
|
|
36
42
|
*.pyd
|
|
37
|
-
|
|
43
|
+
|
|
44
|
+
# Data files
|
|
45
|
+
data/cc100_samples/
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Contributing to skimtoken
|
|
2
|
+
|
|
3
|
+
Thank you for your interest in contributing to skimtoken! We welcome contributions that improve accuracy while maintaining our core principles of minimal memory usage and fast startup times.
|
|
4
|
+
|
|
5
|
+
## Ways to Contribute
|
|
6
|
+
|
|
7
|
+
### 1. Improve Accuracy
|
|
8
|
+
- Test on your language/domain and report accuracy
|
|
9
|
+
- Submit optimized parameters for specific languages
|
|
10
|
+
- Propose better statistical models
|
|
11
|
+
|
|
12
|
+
### 2. Enhance Performance
|
|
13
|
+
- Optimize for lower memory usage
|
|
14
|
+
- Propose changes to speed up tokenization
|
|
15
|
+
- Suggest efficient algorithms or data structures
|
|
16
|
+
|
|
17
|
+
### 3. Report Issues
|
|
18
|
+
- Accuracy problems with specific text types
|
|
19
|
+
- Performance regressions
|
|
20
|
+
- API usability feedback
|
|
21
|
+
|
|
22
|
+
### 4. Documentation
|
|
23
|
+
- Usage examples for different scenarios
|
|
24
|
+
- Tutorials for specific use cases
|
|
25
|
+
- Clarifications and corrections
|
|
26
|
+
|
|
27
|
+
## Development Setup
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
# Clone the repository
|
|
31
|
+
git clone https://github.com/masaishi/skimtoken
|
|
32
|
+
cd skimtoken
|
|
33
|
+
|
|
34
|
+
# Install dependencies with uv
|
|
35
|
+
uv sync
|
|
36
|
+
|
|
37
|
+
# Build development version
|
|
38
|
+
uv run maturin dev --features python
|
|
39
|
+
|
|
40
|
+
# Run tests
|
|
41
|
+
cargo test
|
|
42
|
+
uv run pytest
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Testing Your Changes
|
|
46
|
+
|
|
47
|
+
### 1. Run the Test Suite
|
|
48
|
+
```bash
|
|
49
|
+
# Rust tests
|
|
50
|
+
cargo test
|
|
51
|
+
|
|
52
|
+
# Python tests
|
|
53
|
+
uv run pytest
|
|
54
|
+
|
|
55
|
+
# Benchmarks
|
|
56
|
+
uv run scripts/benchmark.py
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### 2. Check Code Quality
|
|
60
|
+
```bash
|
|
61
|
+
# Python
|
|
62
|
+
uv run ruff format && uv run ruff check --fix && uv run pyright
|
|
63
|
+
|
|
64
|
+
# Rust
|
|
65
|
+
cargo fmt && cargo clippy -- -D warnings
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Submitting Changes
|
|
69
|
+
|
|
70
|
+
### 1. Code Contributions
|
|
71
|
+
|
|
72
|
+
1. Fork the repository
|
|
73
|
+
2. Create a feature branch: `git checkout -b feat/your-feature`
|
|
74
|
+
3. Make your changes
|
|
75
|
+
4. Add tests for new functionality
|
|
76
|
+
5. Ensure all tests pass
|
|
77
|
+
6. Push and create a pull request
|
|
78
|
+
|
|
79
|
+
### 2. Commit Message Format
|
|
80
|
+
|
|
81
|
+
Follow conventional commits:
|
|
82
|
+
- `feat:` New features
|
|
83
|
+
- `fix:` Bug fixes
|
|
84
|
+
- `docs:` Documentation changes
|
|
85
|
+
- `test:` Test additions/changes
|
|
86
|
+
- `perf:` Performance improvements
|
|
87
|
+
- `refactor:` Code refactoring
|
|
88
|
+
|
|
89
|
+
## Questions?
|
|
90
|
+
|
|
91
|
+
Feel free to:
|
|
92
|
+
- Open an issue for discussion
|
|
93
|
+
- Ask in pull request comments
|
|
94
|
+
- Reach out to maintainers
|
|
95
|
+
|
|
96
|
+
Thank you for helping make skimtoken better!
|
|
@@ -29,6 +29,56 @@ version = "0.2.21"
|
|
|
29
29
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
30
|
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
|
31
31
|
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "anstream"
|
|
34
|
+
version = "0.6.19"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
|
|
37
|
+
dependencies = [
|
|
38
|
+
"anstyle",
|
|
39
|
+
"anstyle-parse",
|
|
40
|
+
"anstyle-query",
|
|
41
|
+
"anstyle-wincon",
|
|
42
|
+
"colorchoice",
|
|
43
|
+
"is_terminal_polyfill",
|
|
44
|
+
"utf8parse",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[[package]]
|
|
48
|
+
name = "anstyle"
|
|
49
|
+
version = "1.0.11"
|
|
50
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
51
|
+
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
|
|
52
|
+
|
|
53
|
+
[[package]]
|
|
54
|
+
name = "anstyle-parse"
|
|
55
|
+
version = "0.2.7"
|
|
56
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
57
|
+
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
|
58
|
+
dependencies = [
|
|
59
|
+
"utf8parse",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
[[package]]
|
|
63
|
+
name = "anstyle-query"
|
|
64
|
+
version = "1.1.3"
|
|
65
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
66
|
+
checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
|
|
67
|
+
dependencies = [
|
|
68
|
+
"windows-sys",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[[package]]
|
|
72
|
+
name = "anstyle-wincon"
|
|
73
|
+
version = "3.0.9"
|
|
74
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
75
|
+
checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
|
|
76
|
+
dependencies = [
|
|
77
|
+
"anstyle",
|
|
78
|
+
"once_cell_polyfill",
|
|
79
|
+
"windows-sys",
|
|
80
|
+
]
|
|
81
|
+
|
|
32
82
|
[[package]]
|
|
33
83
|
name = "atty"
|
|
34
84
|
version = "0.2.14"
|
|
@@ -52,6 +102,58 @@ version = "1.0.1"
|
|
|
52
102
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
53
103
|
checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
|
|
54
104
|
|
|
105
|
+
[[package]]
|
|
106
|
+
name = "clap"
|
|
107
|
+
version = "4.5.40"
|
|
108
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
109
|
+
checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f"
|
|
110
|
+
dependencies = [
|
|
111
|
+
"clap_builder",
|
|
112
|
+
"clap_derive",
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
[[package]]
|
|
116
|
+
name = "clap_builder"
|
|
117
|
+
version = "4.5.40"
|
|
118
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
119
|
+
checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e"
|
|
120
|
+
dependencies = [
|
|
121
|
+
"anstream",
|
|
122
|
+
"anstyle",
|
|
123
|
+
"clap_lex",
|
|
124
|
+
"strsim",
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
[[package]]
|
|
128
|
+
name = "clap_derive"
|
|
129
|
+
version = "4.5.40"
|
|
130
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
131
|
+
checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce"
|
|
132
|
+
dependencies = [
|
|
133
|
+
"heck",
|
|
134
|
+
"proc-macro2",
|
|
135
|
+
"quote",
|
|
136
|
+
"syn",
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
[[package]]
|
|
140
|
+
name = "clap_lex"
|
|
141
|
+
version = "0.7.5"
|
|
142
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
143
|
+
checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
|
|
144
|
+
|
|
145
|
+
[[package]]
|
|
146
|
+
name = "colorchoice"
|
|
147
|
+
version = "1.0.4"
|
|
148
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
149
|
+
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
|
|
150
|
+
|
|
151
|
+
[[package]]
|
|
152
|
+
name = "equivalent"
|
|
153
|
+
version = "1.0.2"
|
|
154
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
155
|
+
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
|
156
|
+
|
|
55
157
|
[[package]]
|
|
56
158
|
name = "hashbrown"
|
|
57
159
|
version = "0.14.5"
|
|
@@ -62,6 +164,12 @@ dependencies = [
|
|
|
62
164
|
"allocator-api2",
|
|
63
165
|
]
|
|
64
166
|
|
|
167
|
+
[[package]]
|
|
168
|
+
name = "hashbrown"
|
|
169
|
+
version = "0.15.4"
|
|
170
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
171
|
+
checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
|
|
172
|
+
|
|
65
173
|
[[package]]
|
|
66
174
|
name = "heck"
|
|
67
175
|
version = "0.5.0"
|
|
@@ -77,12 +185,28 @@ dependencies = [
|
|
|
77
185
|
"libc",
|
|
78
186
|
]
|
|
79
187
|
|
|
188
|
+
[[package]]
|
|
189
|
+
name = "indexmap"
|
|
190
|
+
version = "2.10.0"
|
|
191
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
192
|
+
checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
|
|
193
|
+
dependencies = [
|
|
194
|
+
"equivalent",
|
|
195
|
+
"hashbrown 0.15.4",
|
|
196
|
+
]
|
|
197
|
+
|
|
80
198
|
[[package]]
|
|
81
199
|
name = "indoc"
|
|
82
200
|
version = "2.0.6"
|
|
83
201
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
202
|
checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
|
|
85
203
|
|
|
204
|
+
[[package]]
|
|
205
|
+
name = "is_terminal_polyfill"
|
|
206
|
+
version = "1.70.1"
|
|
207
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
208
|
+
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
|
|
209
|
+
|
|
86
210
|
[[package]]
|
|
87
211
|
name = "lazy_static"
|
|
88
212
|
version = "1.5.0"
|
|
@@ -116,6 +240,12 @@ version = "1.21.3"
|
|
|
116
240
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
117
241
|
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|
118
242
|
|
|
243
|
+
[[package]]
|
|
244
|
+
name = "once_cell_polyfill"
|
|
245
|
+
version = "1.70.1"
|
|
246
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
247
|
+
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
|
|
248
|
+
|
|
119
249
|
[[package]]
|
|
120
250
|
name = "portable-atomic"
|
|
121
251
|
version = "1.11.1"
|
|
@@ -231,17 +361,55 @@ version = "0.8.5"
|
|
|
231
361
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
232
362
|
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
|
233
363
|
|
|
364
|
+
[[package]]
|
|
365
|
+
name = "serde"
|
|
366
|
+
version = "1.0.219"
|
|
367
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
368
|
+
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
|
|
369
|
+
dependencies = [
|
|
370
|
+
"serde_derive",
|
|
371
|
+
]
|
|
372
|
+
|
|
373
|
+
[[package]]
|
|
374
|
+
name = "serde_derive"
|
|
375
|
+
version = "1.0.219"
|
|
376
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
377
|
+
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
|
|
378
|
+
dependencies = [
|
|
379
|
+
"proc-macro2",
|
|
380
|
+
"quote",
|
|
381
|
+
"syn",
|
|
382
|
+
]
|
|
383
|
+
|
|
384
|
+
[[package]]
|
|
385
|
+
name = "serde_spanned"
|
|
386
|
+
version = "0.6.9"
|
|
387
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
388
|
+
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
|
|
389
|
+
dependencies = [
|
|
390
|
+
"serde",
|
|
391
|
+
]
|
|
392
|
+
|
|
234
393
|
[[package]]
|
|
235
394
|
name = "skimtoken"
|
|
236
|
-
version = "0.
|
|
395
|
+
version = "0.2.0"
|
|
237
396
|
dependencies = [
|
|
238
397
|
"atty",
|
|
398
|
+
"clap",
|
|
239
399
|
"lazy_static",
|
|
240
400
|
"pyo3",
|
|
241
401
|
"regex",
|
|
402
|
+
"serde",
|
|
403
|
+
"toml",
|
|
242
404
|
"whatlang",
|
|
243
405
|
]
|
|
244
406
|
|
|
407
|
+
[[package]]
|
|
408
|
+
name = "strsim"
|
|
409
|
+
version = "0.11.1"
|
|
410
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
411
|
+
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
412
|
+
|
|
245
413
|
[[package]]
|
|
246
414
|
name = "syn"
|
|
247
415
|
version = "2.0.104"
|
|
@@ -259,6 +427,47 @@ version = "0.13.2"
|
|
|
259
427
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
260
428
|
checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
|
|
261
429
|
|
|
430
|
+
[[package]]
|
|
431
|
+
name = "toml"
|
|
432
|
+
version = "0.8.23"
|
|
433
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
434
|
+
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
|
|
435
|
+
dependencies = [
|
|
436
|
+
"serde",
|
|
437
|
+
"serde_spanned",
|
|
438
|
+
"toml_datetime",
|
|
439
|
+
"toml_edit",
|
|
440
|
+
]
|
|
441
|
+
|
|
442
|
+
[[package]]
|
|
443
|
+
name = "toml_datetime"
|
|
444
|
+
version = "0.6.11"
|
|
445
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
446
|
+
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
|
|
447
|
+
dependencies = [
|
|
448
|
+
"serde",
|
|
449
|
+
]
|
|
450
|
+
|
|
451
|
+
[[package]]
|
|
452
|
+
name = "toml_edit"
|
|
453
|
+
version = "0.22.27"
|
|
454
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
455
|
+
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
|
|
456
|
+
dependencies = [
|
|
457
|
+
"indexmap",
|
|
458
|
+
"serde",
|
|
459
|
+
"serde_spanned",
|
|
460
|
+
"toml_datetime",
|
|
461
|
+
"toml_write",
|
|
462
|
+
"winnow",
|
|
463
|
+
]
|
|
464
|
+
|
|
465
|
+
[[package]]
|
|
466
|
+
name = "toml_write"
|
|
467
|
+
version = "0.1.2"
|
|
468
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
469
|
+
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
|
|
470
|
+
|
|
262
471
|
[[package]]
|
|
263
472
|
name = "unicode-ident"
|
|
264
473
|
version = "1.0.18"
|
|
@@ -271,6 +480,12 @@ version = "0.2.4"
|
|
|
271
480
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
272
481
|
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
|
273
482
|
|
|
483
|
+
[[package]]
|
|
484
|
+
name = "utf8parse"
|
|
485
|
+
version = "0.2.2"
|
|
486
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
487
|
+
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
|
488
|
+
|
|
274
489
|
[[package]]
|
|
275
490
|
name = "version_check"
|
|
276
491
|
version = "0.9.5"
|
|
@@ -283,7 +498,7 @@ version = "0.16.4"
|
|
|
283
498
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
284
499
|
checksum = "471d1c1645d361eb782a1650b1786a8fb58dd625e681a04c09f5ff7c8764a7b0"
|
|
285
500
|
dependencies = [
|
|
286
|
-
"hashbrown",
|
|
501
|
+
"hashbrown 0.14.5",
|
|
287
502
|
"once_cell",
|
|
288
503
|
]
|
|
289
504
|
|
|
@@ -309,6 +524,88 @@ version = "0.4.0"
|
|
|
309
524
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
310
525
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
|
311
526
|
|
|
527
|
+
[[package]]
|
|
528
|
+
name = "windows-sys"
|
|
529
|
+
version = "0.59.0"
|
|
530
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
531
|
+
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
|
532
|
+
dependencies = [
|
|
533
|
+
"windows-targets",
|
|
534
|
+
]
|
|
535
|
+
|
|
536
|
+
[[package]]
|
|
537
|
+
name = "windows-targets"
|
|
538
|
+
version = "0.52.6"
|
|
539
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
540
|
+
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
|
541
|
+
dependencies = [
|
|
542
|
+
"windows_aarch64_gnullvm",
|
|
543
|
+
"windows_aarch64_msvc",
|
|
544
|
+
"windows_i686_gnu",
|
|
545
|
+
"windows_i686_gnullvm",
|
|
546
|
+
"windows_i686_msvc",
|
|
547
|
+
"windows_x86_64_gnu",
|
|
548
|
+
"windows_x86_64_gnullvm",
|
|
549
|
+
"windows_x86_64_msvc",
|
|
550
|
+
]
|
|
551
|
+
|
|
552
|
+
[[package]]
|
|
553
|
+
name = "windows_aarch64_gnullvm"
|
|
554
|
+
version = "0.52.6"
|
|
555
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
556
|
+
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
|
557
|
+
|
|
558
|
+
[[package]]
|
|
559
|
+
name = "windows_aarch64_msvc"
|
|
560
|
+
version = "0.52.6"
|
|
561
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
562
|
+
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
|
563
|
+
|
|
564
|
+
[[package]]
|
|
565
|
+
name = "windows_i686_gnu"
|
|
566
|
+
version = "0.52.6"
|
|
567
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
568
|
+
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
|
569
|
+
|
|
570
|
+
[[package]]
|
|
571
|
+
name = "windows_i686_gnullvm"
|
|
572
|
+
version = "0.52.6"
|
|
573
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
574
|
+
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
|
575
|
+
|
|
576
|
+
[[package]]
|
|
577
|
+
name = "windows_i686_msvc"
|
|
578
|
+
version = "0.52.6"
|
|
579
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
580
|
+
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
|
581
|
+
|
|
582
|
+
[[package]]
|
|
583
|
+
name = "windows_x86_64_gnu"
|
|
584
|
+
version = "0.52.6"
|
|
585
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
586
|
+
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
|
587
|
+
|
|
588
|
+
[[package]]
|
|
589
|
+
name = "windows_x86_64_gnullvm"
|
|
590
|
+
version = "0.52.6"
|
|
591
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
592
|
+
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
|
593
|
+
|
|
594
|
+
[[package]]
|
|
595
|
+
name = "windows_x86_64_msvc"
|
|
596
|
+
version = "0.52.6"
|
|
597
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
598
|
+
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
|
599
|
+
|
|
600
|
+
[[package]]
|
|
601
|
+
name = "winnow"
|
|
602
|
+
version = "0.7.11"
|
|
603
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
604
|
+
checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd"
|
|
605
|
+
dependencies = [
|
|
606
|
+
"memchr",
|
|
607
|
+
]
|
|
608
|
+
|
|
312
609
|
[[package]]
|
|
313
610
|
name = "zerocopy"
|
|
314
611
|
version = "0.8.26"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "skimtoken"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
authors = ["masaishi <mwishiha@ucsc.edu>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -23,6 +23,9 @@ whatlang = "0.16"
|
|
|
23
23
|
atty = "0.2"
|
|
24
24
|
lazy_static = "1.4"
|
|
25
25
|
regex = "1.10"
|
|
26
|
+
toml = "0.8"
|
|
27
|
+
serde = { version = "1.0", features = ["derive"] }
|
|
28
|
+
clap = { version = "4.5", features = ["derive"] }
|
|
26
29
|
|
|
27
30
|
[dependencies.pyo3]
|
|
28
31
|
version = "0.25.0"
|