tokforge 0.1.0__tar.gz

tokforge-0.1.0/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

tokforge-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,113 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: -D warnings
+
+jobs:
+  check:
+    name: fmt, clippy, tests (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo fmt --all -- --check
+      - run: cargo clippy --all-targets --locked -- -D warnings
+      # Reference and differential suites print SKIPPED here: model files are gitignored.
+      - run: cargo test --all-targets --locked
+
+  reference:
+    name: reference artifacts and oracle goldens
+    runs-on: ubuntu-latest
+    needs: check
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Fetch pinned artifacts and oracle files
+        run: python -I models/fetch.py
+      - name: Verify committed goldens are reproducible
+        run: |
+          python -m pip install -r tests/fixtures/golden/requirements.txt
+          python -I tests/fixtures/golden/generate.py --check
+      - name: Reference and differential suites must run, not skip
+        env:
+          FAST_TOKENIZER_REQUIRE_MODELS: "1"
+        run: cargo test --release --locked --test reference --test bytelevel -- --show-output
+      - name: Python package matches tiktoken on the real GPT-2 artifact
+        run: |
+          python -m pip install pytest ./python
+          python -m pytest python/tests -q -rs
+
+  python:
+    name: python wheel (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs: check
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo clippy -p tokforge-python --all-targets --locked -- -D warnings
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          args: --release --locked --out dist --manifest-path python/Cargo.toml
+          manylinux: auto
+      - name: Install the wheel and run the Python tests
+        shell: bash
+        run: |
+          python -m pip install pytest
+          python -m pip install --no-index --find-links dist tokforge
+          python -m pytest python/tests -q -rs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.os }}
+          path: dist/*.whl
+
+  release-binaries:
+    name: release binary (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs: check
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            artifact: tokforge-linux-x86_64
+            path: target/release/tokforge
+          - os: windows-latest
+            artifact: tokforge-windows-x86_64
+            path: target/release/tokforge.exe
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build --release --locked
+      - run: ${{ matrix.path }} info
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact }}
+          path: ${{ matrix.path }}

tokforge-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,127 @@
+# Builds abi3 wheels and an sdist, and publishes them to PyPI through trusted
+# publishing (OIDC, no API token) when a v* tag is pushed. Pull requests that
+# touch packaging, and manual runs, build and test everything without
+# publishing.
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+  pull_request:
+    paths:
+      - .github/workflows/release.yml
+      - python/**
+      - Cargo.toml
+      - Cargo.lock
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  version:
+    name: tag matches crate version
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - shell: bash
+        run: |
+          crate=$(grep -m1 '^version' python/Cargo.toml | cut -d'"' -f2)
+          echo "tag ${GITHUB_REF_NAME#v}, python/Cargo.toml $crate"
+          test "${GITHUB_REF_NAME#v}" = "$crate"
+
+  wheels:
+    name: wheel (${{ matrix.os }} ${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { os: ubuntu-latest, target: x86_64, test: true }
+          - { os: ubuntu-latest, target: aarch64, test: false }
+          - { os: windows-latest, target: x64, test: true }
+          - { os: macos-latest, target: aarch64, test: true }
+          - { os: macos-latest, target: x86_64, test: false }
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          target: ${{ matrix.target }}
+          args: --release --locked --out dist --manifest-path python/Cargo.toml
+          manylinux: auto
+      # Cross-compiled wheels cannot run on the build machine; the native ones
+      # cover the same code.
+      - name: Install the wheel and run the Python tests
+        if: matrix.test
+        shell: bash
+        run: |
+          python -m pip install pytest
+          python -m pip install --no-index --find-links dist tokforge
+          python -m pytest python/tests -q -rs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-${{ matrix.os }}-${{ matrix.target }}
+          path: dist/*.whl
+
+  sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist --manifest-path python/Cargo.toml
+      # The sdist has to carry the parent crate it depends on by path, so
+      # prove it builds on its own, away from the checkout.
+      - name: Build and import the sdist in isolation
+        shell: bash
+        run: |
+          python -m venv /tmp/sdist-check
+          cd /tmp
+          /tmp/sdist-check/bin/pip install "$GITHUB_WORKSPACE"/dist/*.tar.gz
+          /tmp/sdist-check/bin/python -c "import tokforge; print(tokforge.Tokenizer)"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist-sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [version, wheels, sdist]
+    if: startsWith(github.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/tokforge
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - run: ls -l dist
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: GitHub release
+    runs-on: ubuntu-latest
+    needs: publish
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create "$GITHUB_REF_NAME" dist/* --repo "$GITHUB_REPOSITORY" --verify-tag --generate-notes

tokforge-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+/target/
+/.setup/
+*.log
+.env
+.env.*
+!.env.example
+/models/*.json
+!/models/README.md
+/data/
+!/models/MANIFEST.json
+/models/oracle/
+
+# Python
+__pycache__/
+.pytest_cache/
+*.pyd
+*.so

tokforge-0.1.0/AGENTS.md

@@ -0,0 +1,20 @@
+# Project priorities
+
+The user's primary goal is a performance-oriented Rust tokenizer and document
+preparation engine, while learning Rust. Treat runtime efficiency and bounded
+resource use as core requirements, alongside correctness.
+
+- Measure performance in release builds. Read PERFORMANCE.md and use the
+  benchmark runner before and after changes to hot paths. Keep the workload,
+  artifact, toolchain, and machine comparable; report limits of synthetic data.
+- Prefer algorithmic improvements, fewer allocations/copies, reused model state,
+  bounded CPU work, and buffered I/O. Keep CPU work off async runtime threads.
+- Preserve verified token budgets, original source byte ranges, Unicode scalar
+  boundaries, deterministic special-token behavior, ordering, and cancellation.
+- Keep general no-fit fallback correct for nonmonotonic token counts; do not
+  silently assume that longer text always produces more tokens.
+- Add caches, more concurrency, native CPU flags, or unsafe code only when
+  profiling and benchmarks support the change. Explain important Rust ownership
+  and concurrency choices briefly so the user can learn from the implementation.
+- Validate changes with cargo fmt, cargo clippy --all-targets -- -D warnings,
+  and the relevant correctness/integration tests.