tokensim 1.0.0__tar.gz

tokensim-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,49 @@
+name: ci
+on:
+  push:
+  pull_request:
+
+jobs:
+  versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: version lockstep
+        run: bash scripts/check_versions.sh
+
+  build-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        compiler: [gcc, clang]
+    steps:
+      - uses: actions/checkout@v5
+      - name: configure
+        run: |
+          if [ "${{ matrix.compiler }}" = "clang" ]; then
+            cmake -S . -B build-clang -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
+          else
+            cmake -S . -B build
+          fi
+      - name: build
+        run: |
+          if [ "${{ matrix.compiler }}" = "clang" ]; then
+            cmake --build build-clang -j 4
+          else
+            cmake --build build -j 4
+          fi
+      - name: model-free smoke ctest
+        run: |
+          B=build; [ "${{ matrix.compiler }}" = "clang" ] && B=build-clang
+          cd $B && ctest -R "test_unicode|test_json|test_bpe_short|test_cls_table|test_scheme_guard|test_avx512_probe|test_spm_tie_rule|test_decode" --output-on-failure
+
+  avx512-sde:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: build
+        run: cmake -S . -B build && cmake --build build -j 4
+      - name: SDE clone + the avx512 battery
+        run: |
+          git clone --depth 1 https://github.com/rapidfuzz/intel-sde /tmp/sde-mirror
+          bash scripts/check_avx512_sde.sh

tokensim-1.0.0/.github/workflows/pages.yml

@@ -0,0 +1,126 @@
+name: pages
+on:
+  push:
+    branches: [main]
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+concurrency:
+  group: pages
+  cancel-in-progress: true
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: install pandoc
+        run: sudo apt-get update && sudo apt-get install -y pandoc
+      - name: render the markdown
+        run: |
+          mkdir -p site
+          for f in README.md TECHNICAL-REPORT.md BENCHMARKS.md; do
+            pandoc "$f" -f gfm -t html5 -o "site/${f%.md}.frag"
+          done
+          python3 <<'PYEOF'
+          import os
+          css = """
+          body {
+              max-width: 52rem;
+              margin: 2rem auto;
+              padding: 0 1.5rem 0 2rem;
+              font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+              font-size: 16px;
+              line-height: 1.7;
+              color: #3d3d4e;
+              background: #faf6ee;
+          }
+          h1, h2, h3, h4 {
+              color: #2a2a3a;
+              margin-top: 2rem;
+              margin-left: -0.5rem;
+              border-bottom: 1px solid #d8cfc0;
+              padding-bottom: 0.3rem;
+              padding-left: 0.5rem;
+          }
+          h1 { font-size: 2rem; }
+          h2 { font-size: 1.4rem; }
+          h3 { font-size: 1.15rem; }
+          p { padding-left: 5px; }
+          code, pre {
+              background: #f0ece4;
+              border-radius: 6px;
+              font-family: "SF Mono", Consolas, "Liberation Mono", monospace;
+          }
+          code { padding: 0.15em 0.4em; font-size: 0.9em; }
+          pre {
+              padding: 1rem;
+              overflow-x: auto;
+              line-height: 1.45;
+          }
+          pre code { padding: 0; background: none; }
+          table {
+              border-collapse: collapse;
+              width: 100%;
+              margin: 1.5rem 0;
+          }
+          th, td {
+              border: 1px solid #d8cfc0;
+              padding: 0.5rem 0.8rem;
+              text-align: left;
+          }
+          th { background: #f0ece4; font-weight: 600; }
+          blockquote {
+              border-left: 4px solid #8a7a5c;
+              margin: 1rem 0;
+              padding: 0.5rem 1rem;
+              color: #5a5548;
+              background: #f5f0e6;
+              border-radius: 0 6px 6px 0;
+          }
+          a { color: #7a6a3a; }
+          a:hover { color: #5a4a2a; text-decoration: underline; }
+          hr { border: none; border-top: 2px solid #d8cfc0; margin: 2rem 0; }
+          .repo-link {
+              display: inline-block;
+              margin-bottom: 1.5rem;
+              font-size: 0.9em;
+              color: #7a6a3a;
+              text-decoration: none;
+              font-weight: 500;
+          }
+          .repo-link:hover { text-decoration: underline; }
+          """
+          for md in ["README", "TECHNICAL-REPORT", "BENCHMARKS"]:
+              frag = f"site/{md}.frag"
+              if not os.path.isfile(frag):
+                  continue
+              body = open(frag).read()
+              link = '<a class="repo-link" href="https://github.com/SudoEvrything/tokensim">tokensim on GitHub</a>\n'
+              body = body.replace('<h2', link + '<h2', 1)
+              html = f"""<!DOCTYPE html>
+          <html>
+          <head>
+          <meta charset="utf-8">
+          <meta name="viewport" content="width=device-width, initial-scale=1">
+          <style>{css}</style>
+          </head>
+          <body>
+          {body}
+          </body>
+          </html>"""
+              open(f"site/{md}.html", "w").write(html)
+              os.remove(frag)
+          open("site/index.html", "w").write(
+              '<!DOCTYPE html><html><head><meta charset="utf-8">'
+              '<meta http-equiv="refresh" content="0;url=TECHNICAL-REPORT.html">'
+              '</head><body></body></html>')
+          PYEOF
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+      - id: deployment
+        uses: actions/deploy-pages@v4

tokensim-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,72 @@
+name: release
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with: {python-version: "3.12"}
+      - run: pip install build
+      - run: python3 -m build --sdist
+      - uses: actions/upload-artifact@v5
+        with: {name: sdist, path: dist/*.tar.gz}
+
+  wheels:
+    needs: [sdist, versions]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-15]
+    steps:
+      - uses: actions/checkout@v5
+      - name: cibuildwheel
+        uses: pypa/cibuildwheel@v4.3.4
+        env:
+          CIBW_ARCHS_LINUX: x86_64
+          CIBW_ARCHS_MACOS: x86_64
+          CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*"
+          CIBW_SKIP: "*-musllinux_*"
+      - uses: actions/upload-artifact@v5
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - run: bash scripts/check_versions.sh
+
+  publish-pypi:
+    needs: [wheels, sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v5
+        with: {path: artifacts}
+      - name: stage
+        run: |
+          mkdir -p dist
+          find artifacts -type f \( -name "*.whl" -o -name "*.tar.gz" \) -exec cp {} dist/ \;
+          ls -la dist/
+      - name: publish
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+
+  publish-crates:
+    needs: [versions]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - run: bash scripts/sync_crate.sh
+      - name: publish
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: cargo publish --allow-dirty --manifest-path bindings/rust/Cargo.toml || echo "already published at this version"