retok 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retok-0.1.0/.copier-answers.yml +11 -0
- retok-0.1.0/.github/workflows/ci.yml +36 -0
- retok-0.1.0/.github/workflows/release.yml +111 -0
- retok-0.1.0/.github/workflows/update-copier-template.yml +26 -0
- retok-0.1.0/.gitignore +50 -0
- retok-0.1.0/.pre-commit-config.yaml +102 -0
- retok-0.1.0/Cargo.lock +229 -0
- retok-0.1.0/Cargo.toml +20 -0
- retok-0.1.0/LICENSE +21 -0
- retok-0.1.0/PKG-INFO +69 -0
- retok-0.1.0/README.md +45 -0
- retok-0.1.0/pyproject.toml +140 -0
- retok-0.1.0/python/retok/__init__.py +113 -0
- retok-0.1.0/python/retok/_retok.pyi +18 -0
- retok-0.1.0/python/retok/py.typed +0 -0
- retok-0.1.0/src/lib.rs +219 -0
- retok-0.1.0/tests/test_retok.py +94 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changes here will be overwritten by Copier
|
|
2
|
+
_commit: 651b300
|
|
3
|
+
_src_path: gh:audivir/copier-template
|
|
4
|
+
author_email: pypi@audivir.de
|
|
5
|
+
author_name: Tim Hörmann
|
|
6
|
+
description: Regex tokenizer and token encoder/decoder in Rust.
|
|
7
|
+
github_org: audivir
|
|
8
|
+
project_name: retok
|
|
9
|
+
project_slug: retok
|
|
10
|
+
project_type: pyo3
|
|
11
|
+
publishable: true
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
concurrency:
|
|
8
|
+
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
9
|
+
cancel-in-progress: true
|
|
10
|
+
jobs:
|
|
11
|
+
prek:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v7
|
|
15
|
+
- uses: astral-sh/setup-uv@v10.0.1
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- run: uv sync --group dev
|
|
19
|
+
# add mypy to path
|
|
20
|
+
- run: echo "$PWD/.venv/bin" >> "$GITHUB_PATH"
|
|
21
|
+
- uses: j178/prek-action@v3.0.0
|
|
22
|
+
tests:
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
strategy:
|
|
25
|
+
fail-fast: false
|
|
26
|
+
matrix:
|
|
27
|
+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v7
|
|
30
|
+
- uses: astral-sh/setup-uv@v10.0.1
|
|
31
|
+
with:
|
|
32
|
+
python-version: ${{ matrix.python-version }}
|
|
33
|
+
- run: uv sync --group dev --python ${{ matrix.python-version }}
|
|
34
|
+
- run: uv run coverage run -m pytest
|
|
35
|
+
- run: uv run coverage combine || true
|
|
36
|
+
- run: uv run coverage report
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
on:
|
|
3
|
+
release:
|
|
4
|
+
types: [published]
|
|
5
|
+
workflow_dispatch:
|
|
6
|
+
permissions:
|
|
7
|
+
contents: read
|
|
8
|
+
jobs:
|
|
9
|
+
check-version:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v7
|
|
13
|
+
- uses: actions/setup-python@v7
|
|
14
|
+
with:
|
|
15
|
+
python-version: "3.12"
|
|
16
|
+
- name: Check that the release tag matches the package version
|
|
17
|
+
if: github.event_name == 'release'
|
|
18
|
+
run: |
|
|
19
|
+
pkg_version="$(python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["version"])')"
|
|
20
|
+
tag_version="${GITHUB_REF_NAME#v}"
|
|
21
|
+
if [ "$pkg_version" != "$tag_version" ]; then
|
|
22
|
+
echo "::error::Release tag $GITHUB_REF_NAME (version $tag_version) does not match Cargo.toml version ($pkg_version)"
|
|
23
|
+
exit 1
|
|
24
|
+
fi
|
|
25
|
+
linux:
|
|
26
|
+
needs: check-version
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
strategy:
|
|
29
|
+
matrix:
|
|
30
|
+
target: [x86_64, aarch64]
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v7
|
|
33
|
+
- uses: actions/setup-python@v7
|
|
34
|
+
with:
|
|
35
|
+
python-version: "3.12"
|
|
36
|
+
- name: Build wheels
|
|
37
|
+
uses: PyO3/maturin-action@v1
|
|
38
|
+
with:
|
|
39
|
+
target: ${{ matrix.target }}
|
|
40
|
+
args: --release --out dist --find-interpreter
|
|
41
|
+
manylinux: manylinux2014
|
|
42
|
+
- uses: actions/upload-artifact@v7
|
|
43
|
+
with:
|
|
44
|
+
name: wheels-linux-${{ matrix.target }}
|
|
45
|
+
path: dist
|
|
46
|
+
macos:
|
|
47
|
+
needs: check-version
|
|
48
|
+
runs-on: ${{ matrix.runner }}
|
|
49
|
+
strategy:
|
|
50
|
+
matrix:
|
|
51
|
+
include:
|
|
52
|
+
- runner: macos-latest
|
|
53
|
+
target: aarch64
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/checkout@v7
|
|
56
|
+
- uses: actions/setup-python@v7
|
|
57
|
+
with:
|
|
58
|
+
python-version: "3.12"
|
|
59
|
+
- name: Build wheels
|
|
60
|
+
uses: PyO3/maturin-action@v1
|
|
61
|
+
with:
|
|
62
|
+
target: ${{ matrix.target }}
|
|
63
|
+
args: --release --out dist --find-interpreter
|
|
64
|
+
- uses: actions/upload-artifact@v7
|
|
65
|
+
with:
|
|
66
|
+
name: wheels-macos-${{ matrix.target }}
|
|
67
|
+
path: dist
|
|
68
|
+
windows:
|
|
69
|
+
needs: check-version
|
|
70
|
+
runs-on: windows-latest
|
|
71
|
+
steps:
|
|
72
|
+
- uses: actions/checkout@v7
|
|
73
|
+
- uses: actions/setup-python@v7
|
|
74
|
+
with:
|
|
75
|
+
python-version: "3.12"
|
|
76
|
+
- name: Build wheels
|
|
77
|
+
uses: PyO3/maturin-action@v1
|
|
78
|
+
with:
|
|
79
|
+
target: x64
|
|
80
|
+
args: --release --out dist --find-interpreter
|
|
81
|
+
- uses: actions/upload-artifact@v7
|
|
82
|
+
with:
|
|
83
|
+
name: wheels-windows-x64
|
|
84
|
+
path: dist
|
|
85
|
+
sdist:
|
|
86
|
+
needs: check-version
|
|
87
|
+
runs-on: ubuntu-latest
|
|
88
|
+
steps:
|
|
89
|
+
- uses: actions/checkout@v7
|
|
90
|
+
- name: Build sdist
|
|
91
|
+
uses: PyO3/maturin-action@v1
|
|
92
|
+
with:
|
|
93
|
+
command: sdist
|
|
94
|
+
args: --out dist
|
|
95
|
+
- uses: actions/upload-artifact@v7
|
|
96
|
+
with:
|
|
97
|
+
name: wheels-sdist
|
|
98
|
+
path: dist
|
|
99
|
+
publish:
|
|
100
|
+
needs: [linux, macos, windows, sdist]
|
|
101
|
+
runs-on: ubuntu-latest
|
|
102
|
+
environment: pypi
|
|
103
|
+
permissions:
|
|
104
|
+
id-token: write
|
|
105
|
+
steps:
|
|
106
|
+
- uses: actions/download-artifact@v8
|
|
107
|
+
with:
|
|
108
|
+
pattern: wheels-*
|
|
109
|
+
path: dist
|
|
110
|
+
merge-multiple: true
|
|
111
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Update Copier template
|
|
2
|
+
on:
|
|
3
|
+
schedule:
|
|
4
|
+
- cron: "0 6 * * 1" # Monday 06:00 UTC
|
|
5
|
+
workflow_dispatch:
|
|
6
|
+
jobs:
|
|
7
|
+
update:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
permissions:
|
|
10
|
+
contents: write
|
|
11
|
+
pull-requests: write
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v7
|
|
14
|
+
with:
|
|
15
|
+
fetch-depth: 0
|
|
16
|
+
- uses: astral-sh/setup-uv@v10.0.1
|
|
17
|
+
- run: uv tool install copier
|
|
18
|
+
# add copier to path
|
|
19
|
+
- run: echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
20
|
+
- run: copier update --skip-answered --defaults --trust
|
|
21
|
+
- uses: peter-evans/create-pull-request@v8
|
|
22
|
+
with:
|
|
23
|
+
commit-message: Update Copier template
|
|
24
|
+
title: Update Copier template
|
|
25
|
+
branch: update-copier-template
|
|
26
|
+
delete-branch: true
|
retok-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# rust-generated files
|
|
2
|
+
Cargo.lock
|
|
3
|
+
**/*.rs.bk
|
|
4
|
+
/target/
|
|
5
|
+
|
|
6
|
+
# python-generated files
|
|
7
|
+
__pycache__/
|
|
8
|
+
*.py[cod]
|
|
9
|
+
*$py.class
|
|
10
|
+
*.pyd
|
|
11
|
+
*.so
|
|
12
|
+
*.egg-info
|
|
13
|
+
build/
|
|
14
|
+
dist/
|
|
15
|
+
wheels/
|
|
16
|
+
|
|
17
|
+
# build artifacts
|
|
18
|
+
.pdm-build/
|
|
19
|
+
.pdm-python
|
|
20
|
+
|
|
21
|
+
# virtual environments
|
|
22
|
+
.venv/
|
|
23
|
+
venv/
|
|
24
|
+
|
|
25
|
+
# test artifacts
|
|
26
|
+
.benchmarks/
|
|
27
|
+
.coverage*
|
|
28
|
+
.pytest_cache/
|
|
29
|
+
|
|
30
|
+
# lint artifacts
|
|
31
|
+
.mypy_cache/
|
|
32
|
+
.ruff_cache/
|
|
33
|
+
|
|
34
|
+
# uv
|
|
35
|
+
uv.lock
|
|
36
|
+
.python-version
|
|
37
|
+
|
|
38
|
+
# editors
|
|
39
|
+
.idea/
|
|
40
|
+
.vscode/
|
|
41
|
+
|
|
42
|
+
# os
|
|
43
|
+
.DS_Store
|
|
44
|
+
Thumbs.db
|
|
45
|
+
|
|
46
|
+
# environment variables
|
|
47
|
+
.env
|
|
48
|
+
env/
|
|
49
|
+
|
|
50
|
+
# custom exclusions
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# To use:
|
|
2
|
+
#
|
|
3
|
+
# prek run -a
|
|
4
|
+
#
|
|
5
|
+
# Or:
|
|
6
|
+
#
|
|
7
|
+
# prek install # (runs every time you commit in git)
|
|
8
|
+
#
|
|
9
|
+
# To update this file:
|
|
10
|
+
#
|
|
11
|
+
# prek update
|
|
12
|
+
#
|
|
13
|
+
# See https://github.com/j178/prek
|
|
14
|
+
repos:
|
|
15
|
+
# standard hooks
|
|
16
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
17
|
+
rev: v6.0.0
|
|
18
|
+
hooks:
|
|
19
|
+
# order from https://prek.j178.dev/builtin
|
|
20
|
+
- id: trailing-whitespace
|
|
21
|
+
- id: check-added-large-files
|
|
22
|
+
- id: check-case-conflict
|
|
23
|
+
- id: end-of-file-fixer
|
|
24
|
+
# id: file-contents-sorter
|
|
25
|
+
- id: fix-byte-order-marker
|
|
26
|
+
- id: check-json
|
|
27
|
+
- id: check-toml
|
|
28
|
+
- id: check-vcs-permalinks
|
|
29
|
+
- id: check-yaml
|
|
30
|
+
- id: check-xml
|
|
31
|
+
- id: mixed-line-ending
|
|
32
|
+
- id: check-symlinks
|
|
33
|
+
- id: destroyed-symlinks
|
|
34
|
+
- id: check-merge-conflict
|
|
35
|
+
- id: detect-private-key
|
|
36
|
+
# - id: no-commit-to-branch
|
|
37
|
+
- id: check-shebang-scripts-are-executable
|
|
38
|
+
- id: check-executables-have-shebangs
|
|
39
|
+
# others
|
|
40
|
+
- id: debug-statements
|
|
41
|
+
- id: requirements-txt-fixer
|
|
42
|
+
# change tabs to spaces
|
|
43
|
+
- repo: https://github.com/Lucas-C/pre-commit-hooks
|
|
44
|
+
rev: v1.5.6
|
|
45
|
+
hooks:
|
|
46
|
+
- id: remove-tabs
|
|
47
|
+
exclude: ^(docs)|\.go$
|
|
48
|
+
# CMake formatting
|
|
49
|
+
- repo: https://github.com/cheshirekow/cmake-format-precommit
|
|
50
|
+
rev: v0.6.13
|
|
51
|
+
hooks:
|
|
52
|
+
- id: cmake-format
|
|
53
|
+
additional_dependencies: [pyyaml]
|
|
54
|
+
types: [file]
|
|
55
|
+
files: (\.cmake|CMakeLists.txt)(.in)?$
|
|
56
|
+
# yaml formatting
|
|
57
|
+
- repo: https://github.com/google/yamlfmt
|
|
58
|
+
rev: v0.21.0
|
|
59
|
+
hooks:
|
|
60
|
+
- id: yamlfmt
|
|
61
|
+
# Suggested hook if you add a .clang-format file
|
|
62
|
+
# - repo: https://github.com/pre-commit/mirrors-clang-format
|
|
63
|
+
# rev: v22.1.3
|
|
64
|
+
# hooks:
|
|
65
|
+
# - id: clang-format
|
|
66
|
+
# toml formatting
|
|
67
|
+
- repo: https://github.com/tombi-toml/tombi-pre-commit
|
|
68
|
+
rev: v1.8.4
|
|
69
|
+
hooks:
|
|
70
|
+
- id: tombi-format
|
|
71
|
+
# markdown formatting
|
|
72
|
+
- repo: https://github.com/executablebooks/mdformat
|
|
73
|
+
rev: 1.0.0
|
|
74
|
+
hooks:
|
|
75
|
+
- id: mdformat
|
|
76
|
+
- repo: https://github.com/shellcheck-py/shellcheck-py
|
|
77
|
+
rev: v0.11.0.1-1
|
|
78
|
+
hooks:
|
|
79
|
+
- id: shellcheck
|
|
80
|
+
files: ^(\.zshrc|\.bashrc|\.env|.*\.sh)$
|
|
81
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
82
|
+
rev: v0.16.4
|
|
83
|
+
hooks:
|
|
84
|
+
- id: ruff-format
|
|
85
|
+
- id: ruff-check
|
|
86
|
+
- repo: local
|
|
87
|
+
hooks:
|
|
88
|
+
- id: mypy
|
|
89
|
+
name: mypy
|
|
90
|
+
entry: mypy
|
|
91
|
+
language: system
|
|
92
|
+
types: [python]
|
|
93
|
+
require_serial: true
|
|
94
|
+
pass_filenames: false
|
|
95
|
+
args: [python, tests]
|
|
96
|
+
- id: clippy
|
|
97
|
+
name: clippy
|
|
98
|
+
entry: cargo clippy --all-targets --all-features -- -D warnings
|
|
99
|
+
language: system
|
|
100
|
+
types: [rust]
|
|
101
|
+
require_serial: true
|
|
102
|
+
pass_filenames: false
|
retok-0.1.0/Cargo.lock
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "aho-corasick"
|
|
7
|
+
version = "1.1.5"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"memchr",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
[[package]]
|
|
15
|
+
name = "crossbeam-deque"
|
|
16
|
+
version = "0.8.8"
|
|
17
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
18
|
+
checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
|
|
19
|
+
dependencies = [
|
|
20
|
+
"crossbeam-epoch",
|
|
21
|
+
"crossbeam-utils",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[[package]]
|
|
25
|
+
name = "crossbeam-epoch"
|
|
26
|
+
version = "0.9.21"
|
|
27
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
28
|
+
checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"crossbeam-utils",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[[package]]
|
|
34
|
+
name = "crossbeam-utils"
|
|
35
|
+
version = "0.8.23"
|
|
36
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
37
|
+
checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
|
|
38
|
+
|
|
39
|
+
[[package]]
|
|
40
|
+
name = "either"
|
|
41
|
+
version = "1.19.0"
|
|
42
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
43
|
+
checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
44
|
+
|
|
45
|
+
[[package]]
|
|
46
|
+
name = "heck"
|
|
47
|
+
version = "0.5.0"
|
|
48
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
49
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
50
|
+
|
|
51
|
+
[[package]]
|
|
52
|
+
name = "libc"
|
|
53
|
+
version = "0.2.190"
|
|
54
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
55
|
+
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
|
|
56
|
+
|
|
57
|
+
[[package]]
|
|
58
|
+
name = "memchr"
|
|
59
|
+
version = "2.8.3"
|
|
60
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
61
|
+
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
|
62
|
+
|
|
63
|
+
[[package]]
|
|
64
|
+
name = "once_cell"
|
|
65
|
+
version = "1.21.4"
|
|
66
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
67
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
68
|
+
|
|
69
|
+
[[package]]
|
|
70
|
+
name = "portable-atomic"
|
|
71
|
+
version = "1.15.0"
|
|
72
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
73
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
74
|
+
|
|
75
|
+
[[package]]
|
|
76
|
+
name = "proc-macro2"
|
|
77
|
+
version = "1.0.107"
|
|
78
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
79
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
80
|
+
dependencies = [
|
|
81
|
+
"unicode-ident",
|
|
82
|
+
]
|
|
83
|
+
|
|
84
|
+
[[package]]
|
|
85
|
+
name = "pyo3"
|
|
86
|
+
version = "0.29.3"
|
|
87
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
88
|
+
checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655"
|
|
89
|
+
dependencies = [
|
|
90
|
+
"libc",
|
|
91
|
+
"once_cell",
|
|
92
|
+
"portable-atomic",
|
|
93
|
+
"pyo3-build-config",
|
|
94
|
+
"pyo3-ffi",
|
|
95
|
+
"pyo3-macros",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
[[package]]
|
|
99
|
+
name = "pyo3-build-config"
|
|
100
|
+
version = "0.29.3"
|
|
101
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
102
|
+
checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb"
|
|
103
|
+
dependencies = [
|
|
104
|
+
"target-lexicon",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[[package]]
|
|
108
|
+
name = "pyo3-ffi"
|
|
109
|
+
version = "0.29.3"
|
|
110
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
111
|
+
checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330"
|
|
112
|
+
dependencies = [
|
|
113
|
+
"libc",
|
|
114
|
+
"pyo3-build-config",
|
|
115
|
+
]
|
|
116
|
+
|
|
117
|
+
[[package]]
|
|
118
|
+
name = "pyo3-macros"
|
|
119
|
+
version = "0.29.3"
|
|
120
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
121
|
+
checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6"
|
|
122
|
+
dependencies = [
|
|
123
|
+
"proc-macro2",
|
|
124
|
+
"pyo3-macros-backend",
|
|
125
|
+
"quote",
|
|
126
|
+
"syn",
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
[[package]]
|
|
130
|
+
name = "pyo3-macros-backend"
|
|
131
|
+
version = "0.29.3"
|
|
132
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
133
|
+
checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3"
|
|
134
|
+
dependencies = [
|
|
135
|
+
"heck",
|
|
136
|
+
"proc-macro2",
|
|
137
|
+
"quote",
|
|
138
|
+
"syn",
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
[[package]]
|
|
142
|
+
name = "quote"
|
|
143
|
+
version = "1.0.47"
|
|
144
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
145
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
146
|
+
dependencies = [
|
|
147
|
+
"proc-macro2",
|
|
148
|
+
]
|
|
149
|
+
|
|
150
|
+
[[package]]
|
|
151
|
+
name = "rayon"
|
|
152
|
+
version = "1.12.0"
|
|
153
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
154
|
+
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
|
155
|
+
dependencies = [
|
|
156
|
+
"either",
|
|
157
|
+
"rayon-core",
|
|
158
|
+
]
|
|
159
|
+
|
|
160
|
+
[[package]]
|
|
161
|
+
name = "rayon-core"
|
|
162
|
+
version = "1.13.0"
|
|
163
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
164
|
+
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
|
165
|
+
dependencies = [
|
|
166
|
+
"crossbeam-deque",
|
|
167
|
+
"crossbeam-utils",
|
|
168
|
+
]
|
|
169
|
+
|
|
170
|
+
[[package]]
|
|
171
|
+
name = "regex"
|
|
172
|
+
version = "1.13.1"
|
|
173
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
174
|
+
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
|
175
|
+
dependencies = [
|
|
176
|
+
"aho-corasick",
|
|
177
|
+
"memchr",
|
|
178
|
+
"regex-automata",
|
|
179
|
+
"regex-syntax",
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
[[package]]
|
|
183
|
+
name = "regex-automata"
|
|
184
|
+
version = "0.4.18"
|
|
185
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
186
|
+
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
|
187
|
+
dependencies = [
|
|
188
|
+
"aho-corasick",
|
|
189
|
+
"memchr",
|
|
190
|
+
"regex-syntax",
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
[[package]]
|
|
194
|
+
name = "regex-syntax"
|
|
195
|
+
version = "0.8.11"
|
|
196
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
197
|
+
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
|
198
|
+
|
|
199
|
+
[[package]]
|
|
200
|
+
name = "retok"
|
|
201
|
+
version = "0.1.0"
|
|
202
|
+
dependencies = [
|
|
203
|
+
"pyo3",
|
|
204
|
+
"rayon",
|
|
205
|
+
"regex",
|
|
206
|
+
]
|
|
207
|
+
|
|
208
|
+
[[package]]
|
|
209
|
+
name = "syn"
|
|
210
|
+
version = "2.0.119"
|
|
211
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
212
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
213
|
+
dependencies = [
|
|
214
|
+
"proc-macro2",
|
|
215
|
+
"quote",
|
|
216
|
+
"unicode-ident",
|
|
217
|
+
]
|
|
218
|
+
|
|
219
|
+
[[package]]
|
|
220
|
+
name = "target-lexicon"
|
|
221
|
+
version = "0.13.5"
|
|
222
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
223
|
+
checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
|
|
224
|
+
|
|
225
|
+
[[package]]
|
|
226
|
+
name = "unicode-ident"
|
|
227
|
+
version = "1.0.27"
|
|
228
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
229
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
retok-0.1.0/Cargo.toml
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "retok"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
description = "Regex tokenizer and token encoder/decoder in Rust."
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
|
|
8
|
+
[lib]
|
|
9
|
+
crate-type = ["cdylib"]
|
|
10
|
+
name = "_retok"
|
|
11
|
+
|
|
12
|
+
[dependencies]
|
|
13
|
+
pyo3 = { version = "0.29", features = ["extension-module"] }
|
|
14
|
+
rayon = "1"
|
|
15
|
+
regex = "1"
|
|
16
|
+
|
|
17
|
+
[profile.release]
|
|
18
|
+
debug = false
|
|
19
|
+
lto = "fat"
|
|
20
|
+
codegen-units = 1
|
retok-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tim Hörmann
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
retok-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: retok
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Programming Language :: Python :: 3
|
|
5
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
6
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
10
|
+
Classifier: Programming Language :: Rust
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
Requires-Dist: numpy
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Summary: Regex tokenizer and token encoder/decoder in Rust.
|
|
15
|
+
Keywords:
|
|
16
|
+
Author-email: Tim Hörmann <pypi@audivir.de>
|
|
17
|
+
License-Expression: MIT
|
|
18
|
+
Requires-Python: >=3.10, <3.15
|
|
19
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
20
|
+
Project-URL: Homepage, https://github.com/audivir/retok
|
|
21
|
+
Project-URL: Issues, https://github.com/audivir/retok/issues
|
|
22
|
+
Project-URL: Repository, https://github.com/audivir/retok
|
|
23
|
+
|
|
24
|
+
# retok
|
|
25
|
+
|
|
26
|
+
Regex tokenizer and token encoder/decoder written in Rust, exposed to Python via PyO3.
|
|
27
|
+
|
|
28
|
+
## Prerequisites
|
|
29
|
+
|
|
30
|
+
- Python 3.10 or newer
|
|
31
|
+
- Rust toolchain (edition 2024)
|
|
32
|
+
- `maturin`
|
|
33
|
+
|
|
34
|
+
## Installation
|
|
35
|
+
|
|
36
|
+
```shell
|
|
37
|
+
maturin develop
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
The tokenization pattern is passed as argument, as any regex string. Text is enclosed with BOS and
|
|
43
|
+
EOS tokens and padded. With `bos_after_first` the first token stays in front of BOS, and with
|
|
44
|
+
`eos_before_last` the last token stays behind EOS.
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import retok
|
|
48
|
+
|
|
49
|
+
pattern = r"\d+|[A-Za-z]+|[^\s\w]"
|
|
50
|
+
retok.tokenize(pattern, "foo 12, bar!") # ["foo", "12", ",", "bar", "!"]
|
|
51
|
+
|
|
52
|
+
vocab = ["<pad>", "<s>", "</s>", "foo", "bar", "12", ",", "!"]
|
|
53
|
+
token_indices = {t: i for i, t in enumerate(vocab)}
|
|
54
|
+
ids, lengths = retok.encode_with_lengths(
|
|
55
|
+
["foo 12, bar!"],
|
|
56
|
+
min_len=10,
|
|
57
|
+
pattern=pattern,
|
|
58
|
+
token_indices=token_indices,
|
|
59
|
+
pad="<pad>",
|
|
60
|
+
bos="<s>",
|
|
61
|
+
eos="</s>",
|
|
62
|
+
)
|
|
63
|
+
retok.decode(ids, dict(enumerate(vocab)), {"<pad>", "<s>", "</s>"}) # ["foo12,bar!"]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
MIT. See `LICENSE`.
|
|
69
|
+
|
retok-0.1.0/README.md
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# retok
|
|
2
|
+
|
|
3
|
+
Regex tokenizer and token encoder/decoder written in Rust, exposed to Python via PyO3.
|
|
4
|
+
|
|
5
|
+
## Prerequisites
|
|
6
|
+
|
|
7
|
+
- Python 3.10 or newer
|
|
8
|
+
- Rust toolchain (edition 2024)
|
|
9
|
+
- `maturin`
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```shell
|
|
14
|
+
maturin develop
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
The tokenization pattern is passed as argument, as any regex string. Text is enclosed with BOS and
|
|
20
|
+
EOS tokens and padded. With `bos_after_first` the first token stays in front of BOS, and with
|
|
21
|
+
`eos_before_last` the last token stays behind EOS.
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
import retok
|
|
25
|
+
|
|
26
|
+
pattern = r"\d+|[A-Za-z]+|[^\s\w]"
|
|
27
|
+
retok.tokenize(pattern, "foo 12, bar!") # ["foo", "12", ",", "bar", "!"]
|
|
28
|
+
|
|
29
|
+
vocab = ["<pad>", "<s>", "</s>", "foo", "bar", "12", ",", "!"]
|
|
30
|
+
token_indices = {t: i for i, t in enumerate(vocab)}
|
|
31
|
+
ids, lengths = retok.encode_with_lengths(
|
|
32
|
+
["foo 12, bar!"],
|
|
33
|
+
min_len=10,
|
|
34
|
+
pattern=pattern,
|
|
35
|
+
token_indices=token_indices,
|
|
36
|
+
pad="<pad>",
|
|
37
|
+
bos="<s>",
|
|
38
|
+
eos="</s>",
|
|
39
|
+
)
|
|
40
|
+
retok.decode(ids, dict(enumerate(vocab)), {"<pad>", "<s>", "</s>"}) # ["foo12,bar!"]
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## License
|
|
44
|
+
|
|
45
|
+
MIT. See `LICENSE`.
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "retok"
|
|
3
|
+
description = "Regex tokenizer and token encoder/decoder in Rust."
|
|
4
|
+
readme = "README.md"
|
|
5
|
+
requires-python = ">=3.10,<3.15"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [{ name = "Tim Hörmann", email = "pypi@audivir.de" }]
|
|
8
|
+
keywords = []
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Programming Language :: Python :: 3",
|
|
11
|
+
"Programming Language :: Python :: 3.10",
|
|
12
|
+
"Programming Language :: Python :: 3.11",
|
|
13
|
+
"Programming Language :: Python :: 3.12",
|
|
14
|
+
"Programming Language :: Python :: 3.13",
|
|
15
|
+
"Programming Language :: Python :: 3.14",
|
|
16
|
+
"Programming Language :: Rust",
|
|
17
|
+
"Typing :: Typed",
|
|
18
|
+
]
|
|
19
|
+
dependencies = ["numpy"]
|
|
20
|
+
dynamic = ["version"]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://github.com/audivir/retok"
|
|
24
|
+
Issues = "https://github.com/audivir/retok/issues"
|
|
25
|
+
Repository = "https://github.com/audivir/retok"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
|
|
29
|
+
[dependency-groups]
|
|
30
|
+
dev = ["coverage", "mypy", "prek", "pytest", "ruff"]
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["maturin>=1.0,<2.0"]
|
|
34
|
+
build-backend = "maturin"
|
|
35
|
+
|
|
36
|
+
[tool.coverage.run]
|
|
37
|
+
source = ["tests", "retok"]
|
|
38
|
+
command_line = "-m pytest"
|
|
39
|
+
branch = true
|
|
40
|
+
patch = ["subprocess"]
|
|
41
|
+
|
|
42
|
+
[tool.coverage.report]
|
|
43
|
+
omit = ["_types.py", "tests/benchmark.py"]
|
|
44
|
+
exclude_also = [
|
|
45
|
+
"if TYPE_CHECKING:",
|
|
46
|
+
'if __name__ == "__main__"',
|
|
47
|
+
"from __future__ import annotations",
|
|
48
|
+
"@overload",
|
|
49
|
+
"\\.\\.\\.",
|
|
50
|
+
]
|
|
51
|
+
fail_under = 100
|
|
52
|
+
show_missing = true
|
|
53
|
+
|
|
54
|
+
[tool.maturin]
|
|
55
|
+
module-name = "retok._retok"
|
|
56
|
+
python-source = "python"
|
|
57
|
+
|
|
58
|
+
[tool.mypy]
|
|
59
|
+
strict = true
|
|
60
|
+
disable_error_code = "import-untyped"
|
|
61
|
+
enable_error_code = """type-arg,no-untyped-def,redundant-cast,redundant-self,comparison-overlap,
|
|
62
|
+
no-untyped-call,no-any-return,no-any-unimported,unreachable,deprecated,redundant-expr,
|
|
63
|
+
possibly-undefined,truthy-bool,truthy-iterable,ignore-without-code,
|
|
64
|
+
unused-awaitable,unused-ignore,explicit-override,unimported-reveal,mutable-override,explicit-any"""
|
|
65
|
+
|
|
66
|
+
# beyond strict
|
|
67
|
+
warn_unreachable = true
|
|
68
|
+
warn_unused_configs = true
|
|
69
|
+
local_partial_types = true
|
|
70
|
+
|
|
71
|
+
exclude = ["build", "docs"]
|
|
72
|
+
|
|
73
|
+
[tool.pytest.ini_options]
|
|
74
|
+
addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
|
|
75
|
+
filterwarnings = [
|
|
76
|
+
"error",
|
|
77
|
+
"ignore::pytest.PytestCacheWarning",
|
|
78
|
+
"ignore::DeprecationWarning",
|
|
79
|
+
]
|
|
80
|
+
log_cli_level = "INFO"
|
|
81
|
+
minversion = "8.0"
|
|
82
|
+
python_files = ["test_*.py", "end_to_end.py"]
|
|
83
|
+
testpaths = ["tests"]
|
|
84
|
+
xfail_strict = true
|
|
85
|
+
|
|
86
|
+
[tool.ruff]
|
|
87
|
+
line-length = 100
|
|
88
|
+
|
|
89
|
+
[tool.ruff.lint]
|
|
90
|
+
dummy-variable-rgx = "^(dummy_.*|unused_.*|_)$"
|
|
91
|
+
|
|
92
|
+
select = ["ALL"]
|
|
93
|
+
|
|
94
|
+
ignore = [
|
|
95
|
+
"CPY001", # Missing copyright notice at top of file
|
|
96
|
+
"PLC0415", # `import` should be at the top-level of a file
|
|
97
|
+
"ISC001", # implicit-str-concat
|
|
98
|
+
"COM812", # missing-trailing-comma
|
|
99
|
+
"S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue
|
|
100
|
+
"TD003", # Missing issue link on the line following this TODO
|
|
101
|
+
"FIX002", # Line contains TODO, consider resolving the issue
|
|
102
|
+
"FBT001", # Boolean-typed positional argument in function definition
|
|
103
|
+
"FBT002", # Boolean default positional argument
|
|
104
|
+
"FBT003", # Boolean positional value in function call
|
|
105
|
+
"ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `**kwargs`
|
|
106
|
+
"EM101", # Exception must not use a string literal, assign to variable first
|
|
107
|
+
"EM102", # Exception must not use an f-string literal, assign to variable first
|
|
108
|
+
"TRY003", # Avoid specifying long messages outside the exception class
|
|
109
|
+
"TRY300", # Use of else blocks in try/except
|
|
110
|
+
"D105", # Missing docstring in magic method
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
[tool.ruff.lint.pydocstyle]
|
|
114
|
+
convention = "google"
|
|
115
|
+
|
|
116
|
+
[tool.ruff.lint.isort]
|
|
117
|
+
required-imports = ["from __future__ import annotations"]
|
|
118
|
+
split-on-trailing-comma = false
|
|
119
|
+
|
|
120
|
+
[tool.ruff.lint.flake8-tidy-imports]
|
|
121
|
+
ban-relative-imports = "all"
|
|
122
|
+
|
|
123
|
+
[tool.ruff.lint.per-file-ignores]
|
|
124
|
+
# Stub files for C extensions are allowed to have docstrings and overwrite __doc__, __version__, etc.
|
|
125
|
+
"_*.pyi" = ["PYI021", "A"]
|
|
126
|
+
|
|
127
|
+
# conf.py automatically created by Sphinx
|
|
128
|
+
"docs/conf.py" = ["INP001", "D100", "ERA001", "A"]
|
|
129
|
+
|
|
130
|
+
# Test files do not need to have docstrings at each test function
|
|
131
|
+
# S101: assert, PLR2004: magic value comparison, T20: print statements
|
|
132
|
+
"**/test_*.py" = ["INP001", "PLR2004", "S101", "D101", "D102", "D103", "T20"]
|
|
133
|
+
"tests/**" = ["INP001", "PLR2004", "S101", "D101", "D102", "D103", "T20"]
|
|
134
|
+
|
|
135
|
+
[tool.vulture]
|
|
136
|
+
ignore_names = ["_*", "unused_*"]
|
|
137
|
+
paths = [".whitelist", "tests", "python"]
|
|
138
|
+
make_whitelist = true
|
|
139
|
+
sort_by_size = true
|
|
140
|
+
verbose = false
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Rust-accelerated regex tokenizer and encoder/decoder."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from retok import _retok
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
|
+
|
|
14
|
+
from numpy.typing import NDArray
|
|
15
|
+
|
|
16
|
+
__all__ = ["decode", "encode", "encode_with_lengths", "tokenize"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def tokenize(pattern: str, text: str) -> list[str]:
|
|
20
|
+
"""Splits text into all non-overlapping matches of regex pattern."""
|
|
21
|
+
return _retok.tokenize(pattern, text)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def encode_with_lengths( # noqa: PLR0913,PLR0917
|
|
25
|
+
texts: Sequence[str],
|
|
26
|
+
min_len: int,
|
|
27
|
+
pattern: str,
|
|
28
|
+
token_indices: Mapping[str, int],
|
|
29
|
+
pad: str,
|
|
30
|
+
bos: str,
|
|
31
|
+
eos: str,
|
|
32
|
+
bos_after_first: bool = False,
|
|
33
|
+
eos_before_last: bool = False,
|
|
34
|
+
) -> tuple[NDArray[np.int32], NDArray[np.int32]]:
|
|
35
|
+
"""Tokenizes, encloses with BOS and EOS, pads, and encodes texts.
|
|
36
|
+
|
|
37
|
+
Rows are padded to the longer of `min_len` and the longest enclosed row.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
texts: Input texts.
|
|
41
|
+
min_len: Minimum padded length.
|
|
42
|
+
pattern: Regex whose non-overlapping matches are the tokens.
|
|
43
|
+
token_indices: Mapping from tokens to indices.
|
|
44
|
+
pad: Padding token.
|
|
45
|
+
bos: Start-of-sequence token.
|
|
46
|
+
eos: End-of-sequence token.
|
|
47
|
+
bos_after_first: Whether the first token stays in front of BOS.
|
|
48
|
+
eos_before_last: Whether the last token stays behind EOS.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
Padded 2-D id array and unpadded row lengths.
|
|
52
|
+
|
|
53
|
+
Raises:
|
|
54
|
+
ValueError: If a text has too few tokens for the chosen enclosing mode.
|
|
55
|
+
KeyError: If a token is missing from `token_indices`.
|
|
56
|
+
"""
|
|
57
|
+
ids, lengths, n, width = _retok.encode_with_lengths(
|
|
58
|
+
list(texts),
|
|
59
|
+
min_len,
|
|
60
|
+
pattern,
|
|
61
|
+
dict(token_indices),
|
|
62
|
+
pad,
|
|
63
|
+
bos,
|
|
64
|
+
eos,
|
|
65
|
+
bos_after_first,
|
|
66
|
+
eos_before_last,
|
|
67
|
+
)
|
|
68
|
+
return (
|
|
69
|
+
np.frombuffer(ids, dtype=np.int32).reshape(n, width),
|
|
70
|
+
np.frombuffer(lengths, dtype=np.int32),
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def encode( # noqa: PLR0913,PLR0917
|
|
75
|
+
texts: Sequence[str],
|
|
76
|
+
min_len: int,
|
|
77
|
+
pattern: str,
|
|
78
|
+
token_indices: Mapping[str, int],
|
|
79
|
+
pad: str,
|
|
80
|
+
bos: str,
|
|
81
|
+
eos: str,
|
|
82
|
+
bos_after_first: bool = False,
|
|
83
|
+
eos_before_last: bool = False,
|
|
84
|
+
) -> NDArray[np.int32]:
|
|
85
|
+
"""Encodes texts to a padded 2-D id array."""
|
|
86
|
+
return encode_with_lengths(
|
|
87
|
+
texts, min_len, pattern, token_indices, pad, bos, eos, bos_after_first, eos_before_last
|
|
88
|
+
)[0]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def decode(
|
|
92
|
+
inputs: NDArray[np.int32],
|
|
93
|
+
decode_map: Mapping[int, str],
|
|
94
|
+
special_tokens: set[str] | None = None,
|
|
95
|
+
) -> list[str]:
|
|
96
|
+
"""Decodes a 2-D id array to strings, dropping special tokens.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
inputs: Encoded sequences.
|
|
100
|
+
decode_map: Mapping from indices to tokens.
|
|
101
|
+
special_tokens: Tokens omitted from the output.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
One decoded string per row.
|
|
105
|
+
|
|
106
|
+
Raises:
|
|
107
|
+
ValueError: If inputs is not 2-D.
|
|
108
|
+
KeyError: If an index is missing from `decode_map`.
|
|
109
|
+
"""
|
|
110
|
+
arr = np.ascontiguousarray(inputs, dtype=np.int32)
|
|
111
|
+
if arr.ndim != 2: # noqa: PLR2004
|
|
112
|
+
raise ValueError("inputs must be 2-D")
|
|
113
|
+
return _retok.decode(arr.tobytes(), arr.shape[1], dict(decode_map), special_tokens)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
def tokenize(pattern: str, text: str) -> list[str]: ...
|
|
2
|
+
def encode_with_lengths(
|
|
3
|
+
texts: list[str],
|
|
4
|
+
min_len: int,
|
|
5
|
+
pattern: str,
|
|
6
|
+
token_indices: dict[str, int],
|
|
7
|
+
pad: str,
|
|
8
|
+
bos: str,
|
|
9
|
+
eos: str,
|
|
10
|
+
bos_after_first: bool = False,
|
|
11
|
+
eos_before_last: bool = False,
|
|
12
|
+
) -> tuple[bytes, bytes, int, int]: ...
|
|
13
|
+
def decode(
|
|
14
|
+
buffer: bytes,
|
|
15
|
+
width: int,
|
|
16
|
+
decode_map: dict[int, str],
|
|
17
|
+
special_tokens: set[str] | None = None,
|
|
18
|
+
) -> list[str]: ...
|
|
File without changes
|
retok-0.1.0/src/lib.rs
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
//! Regex tokenizer and token encoder/decoder.
|
|
2
|
+
|
|
3
|
+
use std::collections::{HashMap, HashSet};
|
|
4
|
+
|
|
5
|
+
use pyo3::exceptions::{PyKeyError, PyValueError};
|
|
6
|
+
use pyo3::prelude::*;
|
|
7
|
+
use pyo3::types::PyBytes;
|
|
8
|
+
use rayon::prelude::*;
|
|
9
|
+
use regex::Regex;
|
|
10
|
+
|
|
11
|
+
/// Failure conditions while encoding or decoding.
|
|
12
|
+
#[derive(Debug)]
|
|
13
|
+
enum EncodeError {
|
|
14
|
+
TooFewTokens,
|
|
15
|
+
UnknownToken(String),
|
|
16
|
+
UnknownIndex(i32),
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
impl From<EncodeError> for PyErr {
|
|
20
|
+
fn from(e: EncodeError) -> Self {
|
|
21
|
+
match e {
|
|
22
|
+
EncodeError::TooFewTokens => PyValueError::new_err("Too few tokens"),
|
|
23
|
+
EncodeError::UnknownToken(t) => PyKeyError::new_err(t),
|
|
24
|
+
EncodeError::UnknownIndex(i) => PyKeyError::new_err(i),
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/// Returns all non-overlapping matches of `re` in `text`.
|
|
30
|
+
fn tokenize_with<'a>(re: &Regex, text: &'a str) -> Vec<&'a str> {
|
|
31
|
+
re.find_iter(text).map(|m| m.as_str()).collect()
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
fn compile(pattern: &str) -> PyResult<Regex> {
|
|
35
|
+
Regex::new(pattern).map_err(|e| PyValueError::new_err(e.to_string()))
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Inserts BOS and EOS ids around `ids`.
|
|
39
|
+
///
|
|
40
|
+
/// With `bos_after_first`, the first id stays in front of BOS. With `eos_before_last`, the last id
|
|
41
|
+
/// stays behind EOS.
|
|
42
|
+
///
|
|
43
|
+
/// # Errors
|
|
44
|
+
///
|
|
45
|
+
/// Returns `Err` if `ids` is too short for the chosen mode.
|
|
46
|
+
fn enclose(
|
|
47
|
+
ids: &[i32],
|
|
48
|
+
bos: i32,
|
|
49
|
+
eos: i32,
|
|
50
|
+
bos_after_first: bool,
|
|
51
|
+
eos_before_last: bool,
|
|
52
|
+
) -> Result<Vec<i32>, EncodeError> {
|
|
53
|
+
if ids.len() < 1 + usize::from(bos_after_first) + usize::from(eos_before_last) {
|
|
54
|
+
return Err(EncodeError::TooFewTokens);
|
|
55
|
+
}
|
|
56
|
+
let (head, rest) = if bos_after_first { ids.split_at(1) } else { ids.split_at(0) };
|
|
57
|
+
let (body, tail) =
|
|
58
|
+
if eos_before_last { rest.split_at(rest.len() - 1) } else { (rest, &[][..]) };
|
|
59
|
+
let mut out = Vec::with_capacity(ids.len() + 2);
|
|
60
|
+
out.extend_from_slice(head);
|
|
61
|
+
out.push(bos);
|
|
62
|
+
out.extend_from_slice(body);
|
|
63
|
+
out.push(eos);
|
|
64
|
+
out.extend_from_slice(tail);
|
|
65
|
+
Ok(out)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/// Pads rows to the longer of `min_len` and the longest row, then flattens row-major.
|
|
69
|
+
///
|
|
70
|
+
/// Returns `(flat, lengths, width, needs_pad)`.
|
|
71
|
+
fn pad_rows(rows: &[Vec<i32>], pad: Option<i32>, min_len: usize) -> (Vec<i32>, Vec<i32>, usize, bool) {
|
|
72
|
+
let lengths: Vec<i32> = rows.iter().map(|r| r.len() as i32).collect();
|
|
73
|
+
let width = rows.iter().map(Vec::len).max().unwrap_or(0).max(min_len);
|
|
74
|
+
let needs_pad = rows.iter().any(|r| r.len() < width);
|
|
75
|
+
let fill = pad.unwrap_or(0);
|
|
76
|
+
let mut flat = Vec::with_capacity(rows.len() * width);
|
|
77
|
+
for r in rows {
|
|
78
|
+
flat.extend_from_slice(r);
|
|
79
|
+
flat.resize(flat.len() + (width - r.len()), fill);
|
|
80
|
+
}
|
|
81
|
+
(flat, lengths, width, needs_pad)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Decodes rows of ids, skipping `special` tokens.
|
|
85
|
+
///
|
|
86
|
+
/// # Errors
|
|
87
|
+
///
|
|
88
|
+
/// Returns `Err` if an id is missing from `decode_map`.
|
|
89
|
+
fn decode_rows(
|
|
90
|
+
flat: &[i32],
|
|
91
|
+
width: usize,
|
|
92
|
+
decode_map: &HashMap<i32, String>,
|
|
93
|
+
special: &HashSet<String>,
|
|
94
|
+
) -> Result<Vec<String>, EncodeError> {
|
|
95
|
+
if width == 0 {
|
|
96
|
+
return Ok(Vec::new());
|
|
97
|
+
}
|
|
98
|
+
flat.par_chunks(width)
|
|
99
|
+
.map(|row| {
|
|
100
|
+
let mut s = String::new();
|
|
101
|
+
for id in row {
|
|
102
|
+
let tok = decode_map.get(id).ok_or(EncodeError::UnknownIndex(*id))?;
|
|
103
|
+
if !special.contains(tok) {
|
|
104
|
+
s.push_str(tok);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
Ok(s)
|
|
108
|
+
})
|
|
109
|
+
.collect()
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
fn i32_bytes(v: &[i32]) -> Vec<u8> {
|
|
113
|
+
v.iter().flat_map(|x| x.to_ne_bytes()).collect()
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/// Splits `text` into all non-overlapping matches of regex `pattern`.
|
|
117
|
+
#[pyfunction]
|
|
118
|
+
fn tokenize(pattern: &str, text: &str) -> PyResult<Vec<String>> {
|
|
119
|
+
let re = compile(pattern)?;
|
|
120
|
+
Ok(tokenize_with(&re, text).into_iter().map(str::to_owned).collect())
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/// Encodes texts to padded ids.
|
|
124
|
+
///
|
|
125
|
+
/// Returns `(ids, lengths, n_rows, width)` where both buffers are native-endian int32.
|
|
126
|
+
#[pyfunction]
|
|
127
|
+
#[pyo3(signature = (texts, min_len, pattern, token_indices, pad, bos, eos, bos_after_first=false, eos_before_last=false))]
|
|
128
|
+
#[allow(clippy::too_many_arguments)]
|
|
129
|
+
fn encode_with_lengths<'py>(
|
|
130
|
+
py: Python<'py>,
|
|
131
|
+
texts: Vec<String>,
|
|
132
|
+
min_len: usize,
|
|
133
|
+
pattern: &str,
|
|
134
|
+
token_indices: HashMap<String, i32>,
|
|
135
|
+
pad: &str,
|
|
136
|
+
bos: &str,
|
|
137
|
+
eos: &str,
|
|
138
|
+
bos_after_first: bool,
|
|
139
|
+
eos_before_last: bool,
|
|
140
|
+
) -> PyResult<(Bound<'py, PyBytes>, Bound<'py, PyBytes>, usize, usize)> {
|
|
141
|
+
let re = compile(pattern)?;
|
|
142
|
+
let get = |t: &str| {
|
|
143
|
+
token_indices.get(t).copied().ok_or_else(|| PyKeyError::new_err(t.to_owned()))
|
|
144
|
+
};
|
|
145
|
+
let (bos_id, eos_id) = (get(bos)?, get(eos)?);
|
|
146
|
+
let pad_id = token_indices.get(pad).copied();
|
|
147
|
+
let n = texts.len();
|
|
148
|
+
let (flat, lengths, width, needs_pad) = py.detach(|| -> Result<_, EncodeError> {
|
|
149
|
+
let rows = texts
|
|
150
|
+
.par_iter()
|
|
151
|
+
.map(|text| {
|
|
152
|
+
let ids = tokenize_with(&re, text)
|
|
153
|
+
.into_iter()
|
|
154
|
+
.map(|t| {
|
|
155
|
+
token_indices
|
|
156
|
+
.get(t)
|
|
157
|
+
.copied()
|
|
158
|
+
.ok_or_else(|| EncodeError::UnknownToken(t.to_owned()))
|
|
159
|
+
})
|
|
160
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
161
|
+
enclose(&ids, bos_id, eos_id, bos_after_first, eos_before_last)
|
|
162
|
+
})
|
|
163
|
+
.collect::<Result<Vec<_>, _>>()?;
|
|
164
|
+
Ok(pad_rows(&rows, pad_id, min_len))
|
|
165
|
+
})?;
|
|
166
|
+
if needs_pad && pad_id.is_none() {
|
|
167
|
+
return Err(PyKeyError::new_err(pad.to_owned()));
|
|
168
|
+
}
|
|
169
|
+
Ok((PyBytes::new(py, &i32_bytes(&flat)), PyBytes::new(py, &i32_bytes(&lengths)), n, width))
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/// Decodes a row-major native-endian int32 buffer with rows of `width` ids.
|
|
173
|
+
#[pyfunction]
|
|
174
|
+
#[pyo3(signature = (buffer, width, decode_map, special_tokens=None))]
|
|
175
|
+
fn decode(
|
|
176
|
+
py: Python<'_>,
|
|
177
|
+
buffer: &[u8],
|
|
178
|
+
width: usize,
|
|
179
|
+
decode_map: HashMap<i32, String>,
|
|
180
|
+
special_tokens: Option<HashSet<String>>,
|
|
181
|
+
) -> PyResult<Vec<String>> {
|
|
182
|
+
let (chunks, remainder) = buffer.as_chunks::<4>();
|
|
183
|
+
if !remainder.is_empty() {
|
|
184
|
+
return Err(PyValueError::new_err("buffer length must be a multiple of 4"));
|
|
185
|
+
}
|
|
186
|
+
let flat: Vec<i32> = chunks.iter().map(|c| i32::from_ne_bytes(*c)).collect();
|
|
187
|
+
let special = special_tokens.unwrap_or_default();
|
|
188
|
+
Ok(py.detach(|| decode_rows(&flat, width, &decode_map, &special))?)
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
#[pymodule]
|
|
192
|
+
fn _retok(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|
193
|
+
m.add_function(wrap_pyfunction!(tokenize, m)?)?;
|
|
194
|
+
m.add_function(wrap_pyfunction!(encode_with_lengths, m)?)?;
|
|
195
|
+
m.add_function(wrap_pyfunction!(decode, m)?)?;
|
|
196
|
+
Ok(())
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
#[cfg(test)]
|
|
200
|
+
mod tests {
|
|
201
|
+
use super::*;
|
|
202
|
+
|
|
203
|
+
#[test]
|
|
204
|
+
fn enclose_modes() {
|
|
205
|
+
let v = [1, 2, 3];
|
|
206
|
+
assert_eq!(enclose(&v, 8, 9, false, false).unwrap(), [8, 1, 2, 3, 9]);
|
|
207
|
+
assert_eq!(enclose(&v, 8, 9, false, true).unwrap(), [8, 1, 2, 9, 3]);
|
|
208
|
+
assert_eq!(enclose(&v, 8, 9, true, false).unwrap(), [1, 8, 2, 3, 9]);
|
|
209
|
+
assert_eq!(enclose(&v, 8, 9, true, true).unwrap(), [1, 8, 2, 9, 3]);
|
|
210
|
+
assert!(enclose(&[1, 2], 8, 9, true, true).is_err());
|
|
211
|
+
assert!(enclose(&[], 8, 9, false, false).is_err());
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
#[test]
|
|
215
|
+
fn tokenize_words_and_numbers() {
|
|
216
|
+
let re = Regex::new(r"\d+|[A-Za-z]+|[^\s\w]").unwrap();
|
|
217
|
+
assert_eq!(tokenize_with(&re, "foo 12, bar!"), ["foo", "12", ",", "bar", "!"]);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Generic tests for the tokenizer, encoder, and decoder."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TypedDict
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import pytest
|
|
9
|
+
import retok
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Kwargs(TypedDict):
|
|
13
|
+
pattern: str
|
|
14
|
+
token_indices: dict[str, int]
|
|
15
|
+
pad: str
|
|
16
|
+
bos: str
|
|
17
|
+
eos: str
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
VOCAB = ["<pad>", "<s>", "</s>", "a", "b", "ab", "c"]
|
|
21
|
+
INDEX = {t: i for i, t in enumerate(VOCAB)}
|
|
22
|
+
KW: Kwargs = {
|
|
23
|
+
"pattern": r"ab|\w",
|
|
24
|
+
"token_indices": INDEX,
|
|
25
|
+
"pad": "<pad>",
|
|
26
|
+
"bos": "<s>",
|
|
27
|
+
"eos": "</s>",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize(
|
|
32
|
+
("pattern", "text", "expected"),
|
|
33
|
+
[
|
|
34
|
+
(r"ab|\w", "abcb", ["ab", "c", "b"]),
|
|
35
|
+
(r"\w|ab", "abcb", ["a", "b", "c", "b"]),
|
|
36
|
+
(r"\d+|[A-Za-z]+|[^\s\w]", "foo 12, bar!", ["foo", "12", ",", "bar", "!"]),
|
|
37
|
+
(r"\w+", "naïve café 日本語", ["naïve", "café", "日本語"]),
|
|
38
|
+
(r"\d", "no digits", []),
|
|
39
|
+
(r"\w+", "", []),
|
|
40
|
+
],
|
|
41
|
+
)
|
|
42
|
+
def test_tokenize(pattern: str, text: str, expected: list[str]) -> None:
|
|
43
|
+
assert retok.tokenize(pattern, text) == expected
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_tokenize_invalid_regex() -> None:
|
|
47
|
+
with pytest.raises(ValueError, match="regex"):
|
|
48
|
+
retok.tokenize("(", "x")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@pytest.mark.parametrize(
|
|
52
|
+
("after_first", "before_last", "expected"),
|
|
53
|
+
[
|
|
54
|
+
(False, False, [1, 3, 5, 6, 2, 0, 0]),
|
|
55
|
+
(False, True, [1, 3, 5, 2, 6, 0, 0]),
|
|
56
|
+
(True, False, [3, 1, 5, 6, 2, 0, 0]),
|
|
57
|
+
(True, True, [3, 1, 5, 2, 6, 0, 0]),
|
|
58
|
+
],
|
|
59
|
+
)
|
|
60
|
+
def test_enclose_modes_and_padding(
|
|
61
|
+
after_first: bool, before_last: bool, expected: list[int]
|
|
62
|
+
) -> None:
|
|
63
|
+
ids, lengths = retok.encode_with_lengths(
|
|
64
|
+
["aabc"], 7, bos_after_first=after_first, eos_before_last=before_last, **KW
|
|
65
|
+
)
|
|
66
|
+
assert ids.tolist() == [expected]
|
|
67
|
+
assert lengths.tolist() == [5]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_rows_grow_beyond_min_len() -> None:
|
|
71
|
+
ids, lengths = retok.encode_with_lengths(["a", "abccc"], 2, **KW)
|
|
72
|
+
assert ids.shape == (2, 6)
|
|
73
|
+
assert lengths.tolist() == [3, 6]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_decode_roundtrip_drops_special_tokens() -> None:
|
|
77
|
+
ids = retok.encode(["abca", "c"], 4, **KW)
|
|
78
|
+
decoded = retok.decode(ids, dict(enumerate(VOCAB)), {"<pad>", "<s>", "</s>"})
|
|
79
|
+
assert decoded == ["abca", "c"]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_errors() -> None:
|
|
83
|
+
updated_kwargs: Kwargs = {**KW, "pattern": r"\w"}
|
|
84
|
+
with pytest.raises(KeyError):
|
|
85
|
+
retok.encode(["z"], 5, **updated_kwargs)
|
|
86
|
+
with pytest.raises(ValueError, match="Too few"):
|
|
87
|
+
retok.encode([""], 5, **KW)
|
|
88
|
+
with pytest.raises(ValueError, match="Too few"):
|
|
89
|
+
retok.encode(["a"], 5, bos_after_first=True, eos_before_last=True, **KW)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_decode_rejects_non_2d() -> None:
|
|
93
|
+
with pytest.raises(ValueError, match="2-D"):
|
|
94
|
+
retok.decode(np.array([1, 2, 3], dtype=np.int32), dict(enumerate(VOCAB)))
|