fastokens 0.1.1__tar.gz

fastokens-0.1.1/.cargo/config.toml

@@ -0,0 +1,2 @@
+[env]
+PCRE2_SYS_STATIC = "1"

fastokens-0.1.1/.github/workflows/build-maturin.yml

@@ -0,0 +1,222 @@
+# This file is autogenerated by maturin v1.8.1
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: build-maturin
+
+on:
+  workflow_call:
+    inputs:
+      publish_pypi:
+        description: "Whether to publish the package to PyPI."
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      PYPI_TOKEN:
+        description: "The PyPI token for publishing."
+        required: true
+
+permissions:
+  contents: read
+  id-token: write
+  attestations: write
+
+jobs:
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+            manylinux: auto
+            ft: true
+          - runner: ubuntu-22.04
+            target: x86
+            manylinux: auto
+            ft: true
+          - runner: ubuntu-22.04
+            target: aarch64
+            manylinux: manylinux_2_28
+            ft: false
+          - runner: ubuntu-22.04
+            target: armv7
+            manylinux: manylinux_2_28
+            ft: false
+          - runner: ubuntu-22.04
+            target: ppc64le
+            manylinux: manylinux_2_28
+            ft: true
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: "false"
+          manylinux: ${{ matrix.platform.manylinux }}
+          before-script-linux: |
+            if command -v apt-get &> /dev/null; then
+                apt-get update && apt-get install libssl-dev libatomic-ops-dev -y
+            elif command -v yum &> /dev/null; then
+                yum install openssl-devel perl-IPC-Cmd -y
+            else
+                echo "Neither apt-get nor yum is installed. Please install a package manager."
+                exit 1
+            fi
+      - name: Build free-threaded wheels
+        uses: PyO3/maturin-action@v1
+        if: ${{ matrix.platform.ft }}
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist -i python3.13t
+          sccache: "false"
+          manylinux: auto
+          before-script-linux: |
+            if command -v apt-get &> /dev/null; then
+                apt-get update && apt-get install libssl-dev libatomic-ops-dev -y
+            elif command -v yum &> /dev/null; then
+                yum install openssl-devel perl-IPC-Cmd -y
+            else
+                echo "Neither apt-get nor yum is installed. Please install a package manager."
+                exit 1
+            fi
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: dist
+
+  musllinux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-22.04
+            target: x86_64
+          - runner: ubuntu-22.04
+            target: x86
+          - runner: ubuntu-22.04
+            target: aarch64
+          - runner: ubuntu-22.04
+            target: armv7
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: "false"
+          manylinux: musllinux_1_2
+      - name: Build free-threaded wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist -i python3.13t
+          sccache: "false"
+          manylinux: musllinux_1_2
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-musllinux-${{ matrix.platform.target }}
+          path: dist
+
+  windows:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+          architecture: x64
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: x64
+          args: --release --out dist --find-interpreter
+          sccache: "false"
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-windows-x64
+          path: dist
+
+  macos:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: macos-15
+            target: x86_64
+          - runner: macos-15
+            target: aarch64
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: "false"
+      - name: Build free-threaded wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist -i python3.13t
+          sccache: "false"
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-macos-${{ matrix.platform.target }}
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: dist
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    needs: [linux, musllinux, windows, macos, sdist]
+    permissions:
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/download-artifact@v4
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-path: "wheels-*/*"
+      - name: Publish to PyPI
+        if: ${{ inputs.publish_pypi }}
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        with:
+          command: upload
+          args: --non-interactive --skip-existing wheels-*/*

fastokens-0.1.1/.github/workflows/build-release-fastokens.yml

@@ -0,0 +1,42 @@
+name: Build and Release fastokens
+
+on:
+  push:
+    branches:
+      - main
+      - master
+    paths:
+      - "python/**"
+      - "src/**"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - ".github/workflows/**"
+    tags:
+      - "*"
+  pull_request:
+    paths:
+      - "python/**"
+      - "src/**"
+      - "Cargo.toml"
+      - "Cargo.lock"
+      - ".github/workflows/**"
+  workflow_dispatch:
+    inputs:
+      publish_pypi:
+        description: "Publish to PyPI?"
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+  id-token: write
+  attestations: write
+
+jobs:
+  build-package:
+    uses: ./.github/workflows/build-maturin.yml
+    with:
+      publish_pypi: ${{ github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.publish_pypi) }}
+    secrets:
+      PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

fastokens-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,91 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Check formatting
+        run: cargo fmt --check
+
+      - name: Run clippy
+        run: cargo clippy -- -D warnings
+
+      - name: Run tests
+        run: cargo test
+
+  install:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install maturin
+        run: pip install maturin
+
+      - name: Build and install wheel
+        run: maturin build --release --out dist && pip install dist/*.whl
+
+      - name: Smoke test
+        run: |
+          python - <<'EOF'
+          import fastokens
+          tok = fastokens.Tokenizer.from_model("Qwen/Qwen3-0.6B")
+          ids = tok.encode("hello world", add_special_tokens=False)
+          assert len(ids.ids) > 0, "encode returned empty ids"
+          decoded = tok.decode(list(ids.ids))
+          assert decoded == "hello world", f"unexpected decode: {decoded!r}"
+          print("smoke test passed")
+          EOF
+
+  patch-transformers:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        transformers: ["4.57.1", "5.3.0"]
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: Swatinem/rust-cache@v2
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install maturin
+        run: pip install maturin
+
+      - name: Build and install wheel
+        run: maturin build --release --out dist && pip install dist/*.whl
+
+      - name: Install transformers ${{ matrix.transformers }} and pytest
+        run: pip install transformers==${{ matrix.transformers }} pytest
+
+      - name: Test patch_transformers
+        run: pytest python/tests/test_patch_transformers.py -v

fastokens-0.1.1/.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.so
+*.dylib
+*.pyd
+
+dist/
+target/