skimtoken 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skimtoken-0.1.2/.github/workflows/ci.yml +57 -0
- skimtoken-0.1.2/.github/workflows/release.yml +65 -0
- skimtoken-0.1.2/.gitignore +37 -0
- skimtoken-0.1.2/Cargo.lock +330 -0
- skimtoken-0.1.2/Cargo.toml +39 -0
- skimtoken-0.1.2/LICENSE +21 -0
- skimtoken-0.1.2/PKG-INFO +185 -0
- skimtoken-0.1.2/README.md +171 -0
- skimtoken-0.1.2/data/test_dataset.jsonl +147 -0
- skimtoken-0.1.2/examples/example.py +12 -0
- skimtoken-0.1.2/pyproject.toml +42 -0
- skimtoken-0.1.2/scripts/benchmark.py +241 -0
- skimtoken-0.1.2/scripts/run_benchmark_multiple.sh +211 -0
- skimtoken-0.1.2/scripts/update_token_counts.py +56 -0
- skimtoken-0.1.2/src/lib.rs +649 -0
- skimtoken-0.1.2/src/main.rs +57 -0
- skimtoken-0.1.2/src/skimtoken/__init__.py +6 -0
- skimtoken-0.1.2/src/skimtoken/__init__.pyi +6 -0
- skimtoken-0.1.2/src/skimtoken/cli.py +44 -0
- skimtoken-0.1.2/tests/test_skimtoken_simple.py +241 -0
- skimtoken-0.1.2/uv.lock +521 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
env:
|
|
10
|
+
CARGO_TERM_COLOR: always
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
ci-checks:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Install Rust
|
|
19
|
+
uses: dtolnay/rust-toolchain@stable
|
|
20
|
+
with:
|
|
21
|
+
components: rustfmt, clippy
|
|
22
|
+
|
|
23
|
+
- name: Set up Python
|
|
24
|
+
uses: actions/setup-python@v5
|
|
25
|
+
with:
|
|
26
|
+
python-version: "3.13"
|
|
27
|
+
|
|
28
|
+
- name: Install uv
|
|
29
|
+
uses: astral-sh/setup-uv@v5
|
|
30
|
+
|
|
31
|
+
- name: Install dependencies
|
|
32
|
+
run: uv sync --dev
|
|
33
|
+
|
|
34
|
+
- name: Build package
|
|
35
|
+
run: uv build
|
|
36
|
+
|
|
37
|
+
# Rust checks
|
|
38
|
+
- name: Check Rust formatting
|
|
39
|
+
run: cargo fmt -- --check
|
|
40
|
+
|
|
41
|
+
- name: Run clippy
|
|
42
|
+
run: cargo clippy -- -D warnings
|
|
43
|
+
|
|
44
|
+
- name: Run Rust tests
|
|
45
|
+
run: cargo test
|
|
46
|
+
|
|
47
|
+
# Python checks
|
|
48
|
+
- name: Run ruff checks
|
|
49
|
+
run: |
|
|
50
|
+
uv run ruff check
|
|
51
|
+
uv run ruff format --check
|
|
52
|
+
|
|
53
|
+
- name: Run Pyright
|
|
54
|
+
run: uv run pyright
|
|
55
|
+
|
|
56
|
+
- name: Run Python tests
|
|
57
|
+
run: uv run pytest
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: write
|
|
10
|
+
id-token: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
release:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Update versions from tag
|
|
19
|
+
run: |
|
|
20
|
+
VERSION=${GITHUB_REF#refs/tags/v}
|
|
21
|
+
echo "Updating to version: $VERSION"
|
|
22
|
+
|
|
23
|
+
# Update pyproject.toml
|
|
24
|
+
sed -i "s/^version = \".*\"/version = \"$VERSION\"/" pyproject.toml
|
|
25
|
+
|
|
26
|
+
# Update Cargo.toml package version only
|
|
27
|
+
awk -v ver="$VERSION" '
|
|
28
|
+
/^\[package\]/ { in_package = 1 }
|
|
29
|
+
in_package && /^version = / && !done {
|
|
30
|
+
sub(/version = ".*"/, "version = \"" ver "\"");
|
|
31
|
+
done = 1
|
|
32
|
+
}
|
|
33
|
+
/^\[/ && !/^\[package\]/ { in_package = 0 }
|
|
34
|
+
{ print }
|
|
35
|
+
' Cargo.toml > Cargo.toml.tmp && mv Cargo.toml.tmp Cargo.toml
|
|
36
|
+
|
|
37
|
+
- name: Install Rust
|
|
38
|
+
uses: dtolnay/rust-toolchain@stable
|
|
39
|
+
|
|
40
|
+
- name: Build Python wheels
|
|
41
|
+
uses: PyO3/maturin-action@v1
|
|
42
|
+
with:
|
|
43
|
+
command: build
|
|
44
|
+
args: --release --sdist --out dist
|
|
45
|
+
manylinux: auto
|
|
46
|
+
|
|
47
|
+
- name: Upload to PyPI
|
|
48
|
+
run: |
|
|
49
|
+
pip install --upgrade maturin
|
|
50
|
+
maturin upload dist/* --non-interactive
|
|
51
|
+
env:
|
|
52
|
+
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
|
|
53
|
+
|
|
54
|
+
- name: Publish to crates.io
|
|
55
|
+
run: cargo publish --allow-dirty
|
|
56
|
+
env:
|
|
57
|
+
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
|
58
|
+
|
|
59
|
+
- name: Create GitHub Release
|
|
60
|
+
uses: softprops/action-gh-release@v2
|
|
61
|
+
with:
|
|
62
|
+
generate_release_notes: true
|
|
63
|
+
files: |
|
|
64
|
+
dist/*.whl
|
|
65
|
+
dist/*.tar.gz
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
*.egg
|
|
7
|
+
*.egg-info/
|
|
8
|
+
dist/
|
|
9
|
+
build/
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
ENV/
|
|
13
|
+
env/
|
|
14
|
+
.coverage*
|
|
15
|
+
coverage.*
|
|
16
|
+
wheels/
|
|
17
|
+
|
|
18
|
+
# Rust
|
|
19
|
+
/target/
|
|
20
|
+
Cargo.lock
|
|
21
|
+
**/*.rs.bak
|
|
22
|
+
|
|
23
|
+
# IDE
|
|
24
|
+
.vscode/
|
|
25
|
+
.idea/
|
|
26
|
+
*.swp
|
|
27
|
+
*.swo
|
|
28
|
+
*~
|
|
29
|
+
|
|
30
|
+
# OS
|
|
31
|
+
.DS_Store
|
|
32
|
+
Thumbs.db
|
|
33
|
+
|
|
34
|
+
# Project specific
|
|
35
|
+
*.dylib
|
|
36
|
+
*.pyd
|
|
37
|
+
skimtoken_exp/
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "ahash"
|
|
7
|
+
version = "0.8.12"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"cfg-if",
|
|
12
|
+
"once_cell",
|
|
13
|
+
"version_check",
|
|
14
|
+
"zerocopy",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[[package]]
|
|
18
|
+
name = "aho-corasick"
|
|
19
|
+
version = "1.1.3"
|
|
20
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
21
|
+
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
|
|
22
|
+
dependencies = [
|
|
23
|
+
"memchr",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[[package]]
|
|
27
|
+
name = "allocator-api2"
|
|
28
|
+
version = "0.2.21"
|
|
29
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
30
|
+
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
|
31
|
+
|
|
32
|
+
[[package]]
|
|
33
|
+
name = "atty"
|
|
34
|
+
version = "0.2.14"
|
|
35
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
36
|
+
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
|
37
|
+
dependencies = [
|
|
38
|
+
"hermit-abi",
|
|
39
|
+
"libc",
|
|
40
|
+
"winapi",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[[package]]
|
|
44
|
+
name = "autocfg"
|
|
45
|
+
version = "1.5.0"
|
|
46
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
47
|
+
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
|
48
|
+
|
|
49
|
+
[[package]]
|
|
50
|
+
name = "cfg-if"
|
|
51
|
+
version = "1.0.1"
|
|
52
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
53
|
+
checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
|
|
54
|
+
|
|
55
|
+
[[package]]
|
|
56
|
+
name = "hashbrown"
|
|
57
|
+
version = "0.14.5"
|
|
58
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
59
|
+
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
|
60
|
+
dependencies = [
|
|
61
|
+
"ahash",
|
|
62
|
+
"allocator-api2",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[[package]]
|
|
66
|
+
name = "heck"
|
|
67
|
+
version = "0.5.0"
|
|
68
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
69
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
70
|
+
|
|
71
|
+
[[package]]
|
|
72
|
+
name = "hermit-abi"
|
|
73
|
+
version = "0.1.19"
|
|
74
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
75
|
+
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
|
|
76
|
+
dependencies = [
|
|
77
|
+
"libc",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
[[package]]
|
|
81
|
+
name = "indoc"
|
|
82
|
+
version = "2.0.6"
|
|
83
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
84
|
+
checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
|
|
85
|
+
|
|
86
|
+
[[package]]
|
|
87
|
+
name = "lazy_static"
|
|
88
|
+
version = "1.5.0"
|
|
89
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
90
|
+
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
|
91
|
+
|
|
92
|
+
[[package]]
|
|
93
|
+
name = "libc"
|
|
94
|
+
version = "0.2.174"
|
|
95
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
96
|
+
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
|
|
97
|
+
|
|
98
|
+
[[package]]
|
|
99
|
+
name = "memchr"
|
|
100
|
+
version = "2.7.5"
|
|
101
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
102
|
+
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
|
|
103
|
+
|
|
104
|
+
[[package]]
|
|
105
|
+
name = "memoffset"
|
|
106
|
+
version = "0.9.1"
|
|
107
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
108
|
+
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
|
109
|
+
dependencies = [
|
|
110
|
+
"autocfg",
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
[[package]]
|
|
114
|
+
name = "once_cell"
|
|
115
|
+
version = "1.21.3"
|
|
116
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
117
|
+
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
|
118
|
+
|
|
119
|
+
[[package]]
|
|
120
|
+
name = "portable-atomic"
|
|
121
|
+
version = "1.11.1"
|
|
122
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
123
|
+
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
|
|
124
|
+
|
|
125
|
+
[[package]]
|
|
126
|
+
name = "proc-macro2"
|
|
127
|
+
version = "1.0.95"
|
|
128
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
129
|
+
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
|
130
|
+
dependencies = [
|
|
131
|
+
"unicode-ident",
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
[[package]]
|
|
135
|
+
name = "pyo3"
|
|
136
|
+
version = "0.25.1"
|
|
137
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
138
|
+
checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a"
|
|
139
|
+
dependencies = [
|
|
140
|
+
"indoc",
|
|
141
|
+
"libc",
|
|
142
|
+
"memoffset",
|
|
143
|
+
"once_cell",
|
|
144
|
+
"portable-atomic",
|
|
145
|
+
"pyo3-build-config",
|
|
146
|
+
"pyo3-ffi",
|
|
147
|
+
"pyo3-macros",
|
|
148
|
+
"unindent",
|
|
149
|
+
]
|
|
150
|
+
|
|
151
|
+
[[package]]
|
|
152
|
+
name = "pyo3-build-config"
|
|
153
|
+
version = "0.25.1"
|
|
154
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
155
|
+
checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598"
|
|
156
|
+
dependencies = [
|
|
157
|
+
"once_cell",
|
|
158
|
+
"target-lexicon",
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
[[package]]
|
|
162
|
+
name = "pyo3-ffi"
|
|
163
|
+
version = "0.25.1"
|
|
164
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
165
|
+
checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c"
|
|
166
|
+
dependencies = [
|
|
167
|
+
"libc",
|
|
168
|
+
"pyo3-build-config",
|
|
169
|
+
]
|
|
170
|
+
|
|
171
|
+
[[package]]
|
|
172
|
+
name = "pyo3-macros"
|
|
173
|
+
version = "0.25.1"
|
|
174
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
175
|
+
checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50"
|
|
176
|
+
dependencies = [
|
|
177
|
+
"proc-macro2",
|
|
178
|
+
"pyo3-macros-backend",
|
|
179
|
+
"quote",
|
|
180
|
+
"syn",
|
|
181
|
+
]
|
|
182
|
+
|
|
183
|
+
[[package]]
|
|
184
|
+
name = "pyo3-macros-backend"
|
|
185
|
+
version = "0.25.1"
|
|
186
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
187
|
+
checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc"
|
|
188
|
+
dependencies = [
|
|
189
|
+
"heck",
|
|
190
|
+
"proc-macro2",
|
|
191
|
+
"pyo3-build-config",
|
|
192
|
+
"quote",
|
|
193
|
+
"syn",
|
|
194
|
+
]
|
|
195
|
+
|
|
196
|
+
[[package]]
|
|
197
|
+
name = "quote"
|
|
198
|
+
version = "1.0.40"
|
|
199
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
200
|
+
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
|
|
201
|
+
dependencies = [
|
|
202
|
+
"proc-macro2",
|
|
203
|
+
]
|
|
204
|
+
|
|
205
|
+
[[package]]
|
|
206
|
+
name = "regex"
|
|
207
|
+
version = "1.11.1"
|
|
208
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
209
|
+
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
|
|
210
|
+
dependencies = [
|
|
211
|
+
"aho-corasick",
|
|
212
|
+
"memchr",
|
|
213
|
+
"regex-automata",
|
|
214
|
+
"regex-syntax",
|
|
215
|
+
]
|
|
216
|
+
|
|
217
|
+
[[package]]
|
|
218
|
+
name = "regex-automata"
|
|
219
|
+
version = "0.4.9"
|
|
220
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
221
|
+
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
|
|
222
|
+
dependencies = [
|
|
223
|
+
"aho-corasick",
|
|
224
|
+
"memchr",
|
|
225
|
+
"regex-syntax",
|
|
226
|
+
]
|
|
227
|
+
|
|
228
|
+
[[package]]
|
|
229
|
+
name = "regex-syntax"
|
|
230
|
+
version = "0.8.5"
|
|
231
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
232
|
+
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
|
|
233
|
+
|
|
234
|
+
[[package]]
|
|
235
|
+
name = "skimtoken"
|
|
236
|
+
version = "0.1.2"
|
|
237
|
+
dependencies = [
|
|
238
|
+
"atty",
|
|
239
|
+
"lazy_static",
|
|
240
|
+
"pyo3",
|
|
241
|
+
"regex",
|
|
242
|
+
"whatlang",
|
|
243
|
+
]
|
|
244
|
+
|
|
245
|
+
[[package]]
|
|
246
|
+
name = "syn"
|
|
247
|
+
version = "2.0.104"
|
|
248
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
249
|
+
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
|
|
250
|
+
dependencies = [
|
|
251
|
+
"proc-macro2",
|
|
252
|
+
"quote",
|
|
253
|
+
"unicode-ident",
|
|
254
|
+
]
|
|
255
|
+
|
|
256
|
+
[[package]]
|
|
257
|
+
name = "target-lexicon"
|
|
258
|
+
version = "0.13.2"
|
|
259
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
260
|
+
checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
|
|
261
|
+
|
|
262
|
+
[[package]]
|
|
263
|
+
name = "unicode-ident"
|
|
264
|
+
version = "1.0.18"
|
|
265
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
266
|
+
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
|
267
|
+
|
|
268
|
+
[[package]]
|
|
269
|
+
name = "unindent"
|
|
270
|
+
version = "0.2.4"
|
|
271
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
272
|
+
checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
|
|
273
|
+
|
|
274
|
+
[[package]]
|
|
275
|
+
name = "version_check"
|
|
276
|
+
version = "0.9.5"
|
|
277
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
278
|
+
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
|
279
|
+
|
|
280
|
+
[[package]]
|
|
281
|
+
name = "whatlang"
|
|
282
|
+
version = "0.16.4"
|
|
283
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
284
|
+
checksum = "471d1c1645d361eb782a1650b1786a8fb58dd625e681a04c09f5ff7c8764a7b0"
|
|
285
|
+
dependencies = [
|
|
286
|
+
"hashbrown",
|
|
287
|
+
"once_cell",
|
|
288
|
+
]
|
|
289
|
+
|
|
290
|
+
[[package]]
|
|
291
|
+
name = "winapi"
|
|
292
|
+
version = "0.3.9"
|
|
293
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
294
|
+
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
|
295
|
+
dependencies = [
|
|
296
|
+
"winapi-i686-pc-windows-gnu",
|
|
297
|
+
"winapi-x86_64-pc-windows-gnu",
|
|
298
|
+
]
|
|
299
|
+
|
|
300
|
+
[[package]]
|
|
301
|
+
name = "winapi-i686-pc-windows-gnu"
|
|
302
|
+
version = "0.4.0"
|
|
303
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
304
|
+
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
|
305
|
+
|
|
306
|
+
[[package]]
|
|
307
|
+
name = "winapi-x86_64-pc-windows-gnu"
|
|
308
|
+
version = "0.4.0"
|
|
309
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
310
|
+
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
|
311
|
+
|
|
312
|
+
[[package]]
|
|
313
|
+
name = "zerocopy"
|
|
314
|
+
version = "0.8.26"
|
|
315
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
316
|
+
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
|
317
|
+
dependencies = [
|
|
318
|
+
"zerocopy-derive",
|
|
319
|
+
]
|
|
320
|
+
|
|
321
|
+
[[package]]
|
|
322
|
+
name = "zerocopy-derive"
|
|
323
|
+
version = "0.8.26"
|
|
324
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
325
|
+
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
|
326
|
+
dependencies = [
|
|
327
|
+
"proc-macro2",
|
|
328
|
+
"quote",
|
|
329
|
+
"syn",
|
|
330
|
+
]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "skimtoken"
|
|
3
|
+
version = "0.1.2"
|
|
4
|
+
edition = "2021"
|
|
5
|
+
authors = ["masaishi <mwishiha@ucsc.edu>"]
|
|
6
|
+
license = "MIT"
|
|
7
|
+
description = "Fast token count estimation library"
|
|
8
|
+
homepage = "https://github.com/masaishi/skimtoken"
|
|
9
|
+
repository = "https://github.com/masaishi/skimtoken"
|
|
10
|
+
documentation = "https://github.com/masaishi/skimtoken"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
|
|
13
|
+
[lib]
|
|
14
|
+
name = "skimtoken"
|
|
15
|
+
crate-type = ["cdylib", "rlib"]
|
|
16
|
+
|
|
17
|
+
[[bin]]
|
|
18
|
+
name = "skimtoken"
|
|
19
|
+
path = "src/main.rs"
|
|
20
|
+
|
|
21
|
+
[dependencies]
|
|
22
|
+
whatlang = "0.16"
|
|
23
|
+
atty = "0.2"
|
|
24
|
+
lazy_static = "1.4"
|
|
25
|
+
regex = "1.10"
|
|
26
|
+
|
|
27
|
+
[dependencies.pyo3]
|
|
28
|
+
version = "0.25.0"
|
|
29
|
+
features = ["extension-module", "abi3-py39"]
|
|
30
|
+
optional = true
|
|
31
|
+
|
|
32
|
+
[features]
|
|
33
|
+
default = []
|
|
34
|
+
python = ["pyo3"]
|
|
35
|
+
|
|
36
|
+
[profile.release]
|
|
37
|
+
lto = true
|
|
38
|
+
opt-level = 3
|
|
39
|
+
codegen-units = 1
|
skimtoken-0.1.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Masamune Ishihara
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|