ggml-python 0.0.30__tar.gz → 0.0.31__tar.gz

ggml_python-0.0.31/.github/workflows/wheels-cuda.yaml

@@ -0,0 +1,130 @@
+name: Wheels CUDA
+
+on: workflow_dispatch
+
+permissions:
+  contents: write
+
+jobs:
+  define_matrix:
+    name: Define Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    defaults:
+      run:
+        shell: pwsh
+
+    steps:
+      - name: Define Job Output
+        id: set-matrix
+        run: |
+          $matrix = @{
+              'os' = @('ubuntu-20.04', 'windows-latest')
+              'pyver' = @("3.10", "3.11", "3.12")
+              'cuda' = @("12.1.1", "12.2.2", "12.3.2")
+              'releasetag' = @("basic")
+          }
+
+          $matrixOut = ConvertTo-Json $matrix -Compress
+          Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT
+
+  build_wheels:
+    name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }}
+    needs: define_matrix
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }}
+    defaults:
+      run:
+        shell: pwsh
+    env:
+      CUDAVER: ${{ matrix.cuda }}
+      AVXVER: ${{ matrix.releasetag }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.pyver }}
+
+      - name: Setup Mamba
+        uses: conda-incubator/setup-miniconda@v2.2.0
+        with:
+          activate-environment: "build"
+          python-version: ${{ matrix.pyver }}
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          add-pip-as-python-dependency: true
+          auto-activate-base: false
+
+      - name: VS Integration Cache
+        id: vs-integration-cache
+        if: runner.os == 'Windows'
+        uses: actions/cache@v3.3.2
+        with:
+          path: ./MSBuildExtensions
+          key: cuda-${{ matrix.cuda }}-vs-integration
+
+      - name: Get Visual Studio Integration
+        if: runner.os == 'Windows' && steps.vs-integration-cache.outputs.cache-hit != 'true'
+        run: |
+          if ($env:CUDAVER -eq '12.1.1') {$x = '12.1.0'} else {$x = $env:CUDAVER}
+          $links = (Invoke-RestMethod 'https://github.com/Jimver/cuda-toolkit/raw/dc0ca7bb29c5a92f7a963d3d5c93f8d59765136a/src/links/windows-links.ts').Trim().split().where({$_ -ne ''})
+          for ($i=$q=0;$i -lt $links.count -and $q -lt 2;$i++) {if ($links[$i] -eq "'$x',") {$q++}}
+          Invoke-RestMethod $links[$i].Trim("'") -OutFile 'cudainstaller.zip'
+          & 'C:\Program Files\7-Zip\7z.exe' e cudainstaller.zip -oMSBuildExtensions -r *\MSBuildExtensions\* > $null
+          Remove-Item 'cudainstaller.zip'
+
+      - name: Install Visual Studio Integration
+        if: runner.os == 'Windows'
+        run: |
+          $y = (gi '.\MSBuildExtensions').fullname + '\*'
+          (gi 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_})
+          $cupath = 'CUDA_PATH_V' + $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','_')
+          echo "$cupath=$env:CONDA_PREFIX" >> $env:GITHUB_ENV
+
+      - name: Install Dependencies
+        env:
+          MAMBA_DOWNLOAD_FAILFAST: "0"
+          MAMBA_NO_LOW_SPEED_LIMIT: "1"
+        run: |
+          $cudaVersion = $env:CUDAVER
+          mamba install -y 'cuda' -c nvidia/label/cuda-$cudaVersion
+          python -m pip install build wheel
+
+      - name: Build Wheel
+        run: |
+          $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')
+          $env:CUDA_PATH = $env:CONDA_PREFIX
+          $env:CUDA_HOME = $env:CONDA_PREFIX
+          $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX
+          if ($IsLinux) {
+            $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH
+          }
+          $env:VERBOSE = '1'
+          $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all'
+          $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS"
+          if ($env:AVXVER -eq 'AVX') {
+            $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
+          }
+          if ($env:AVXVER -eq 'AVX512') {
+            $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on'
+          }
+          if ($env:AVXVER -eq 'basic') {
+            $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
+          }
+          $buildtag = "-cu$cudaVersion"
+          python -m build --wheel
+          Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV
+
+      - uses: softprops/action-gh-release@v1
+        with:
+          files: dist/*
+          tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

ggml_python-0.0.31/.github/workflows/wheels-index.yaml

@@ -0,0 +1,48 @@
+name: Wheels Index
+
+on:
+  # Trigger on any new release
+  release:
+    types: [published]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Build
+        run: |
+          ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
+          ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+$-cu121'
+          ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+$-cu122'
+          ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+$-cu123'
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: "index"
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

{ggml_python-0.0.30 → ggml_python-0.0.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ggml_python
-Version: 0.0.30
+Version: 0.0.31
 Summary: Python bindings for ggml
 Author-Email: Andrei Betlen <abetlen@gmail.com>
 License: MIT
@@ -74,14 +74,14 @@ To build ggml with specific features (ie. OpenBLAS, GPU Support, etc) you can pa
 
 ```bash
 pip install --upgrade pip
-pip install ggml-python --config-settings=cmake.args='-DGGML_CUBLAS=ON'
+pip install ggml-python --config-settings=cmake.args='-DGGML_CUDA=ON'
 ```
 
 ## Options
 
 | Option | Description | Default |
 | --- | --- | --- |
-| `GGML_CUBLAS` | Enable cuBLAS support | `OFF` |
+| `GGML_CUDA` | Enable cuBLAS support | `OFF` |
 | `GGML_CLBLAST` | Enable CLBlast support | `OFF` |
 | `GGML_OPENBLAS` | Enable OpenBLAS support | `OFF` |
 | `GGML_METAL` | Enable Metal support | `OFF` |

{ggml_python-0.0.30 → ggml_python-0.0.31}/README.md

@@ -36,14 +36,14 @@ To build ggml with specific features (ie. OpenBLAS, GPU Support, etc) you can pa
 
 ```bash
 pip install --upgrade pip
-pip install ggml-python --config-settings=cmake.args='-DGGML_CUBLAS=ON'
+pip install ggml-python --config-settings=cmake.args='-DGGML_CUDA=ON'
 ```
 
 ## Options
 
 | Option | Description | Default |
 | --- | --- | --- |
-| `GGML_CUBLAS` | Enable cuBLAS support | `OFF` |
+| `GGML_CUDA` | Enable cuBLAS support | `OFF` |
 | `GGML_CLBLAST` | Enable CLBlast support | `OFF` |
 | `GGML_OPENBLAS` | Enable OpenBLAS support | `OFF` |
 | `GGML_METAL` | Enable Metal support | `OFF` |

{ggml_python-0.0.30 → ggml_python-0.0.31}/examples/replit/main.py

@@ -331,7 +331,7 @@ class ReplitModel:
         self.memory_k = ggml.ggml_new_tensor_1d(ctx, ggml.GGML_TYPE_F16, n_elements)
         self.memory_v = ggml.ggml_new_tensor_1d(ctx, ggml.GGML_TYPE_F16, n_elements)
 
-        if ggml.GGML_USE_CUBLAS:
+        if ggml.GGML_USE_CUDA:
             self.memory_k.contents.backend = ggml.GGML_BACKEND_GPU
             ggml.ggml_cuda_transform_tensor(self.memory_k.contents.data, self.memory_k)
             self.memory_v.contents.backend = ggml.GGML_BACKEND_GPU
@@ -343,7 +343,7 @@ class ReplitModel:
         self.tensors["transformer.norm_f.weight"] = self.norm_f_weight
 
         self.mem_per_token = 0
-        if ggml.GGML_USE_CUBLAS:
+        if ggml.GGML_USE_CUDA:
             self.eval_buffer = CudaContextBuffer()
         else:
             self.eval_buffer = CpuContextBuffer()
@@ -382,7 +382,7 @@ class ReplitModel:
     def __del__(self):
         ggml.ggml_free(self.ctx)
 
-        if ggml.GGML_USE_CUBLAS:
+        if ggml.GGML_USE_CUDA:
             for tensor in self.tensors.values():
                 if tensor.contents.backend == ggml.GGML_BACKEND_GPU:
                     ggml.ggml_cuda_free_data(tensor)
@@ -469,7 +469,7 @@ class ReplitModel:
 
         inpL = ggml.ggml_get_rows(ctx0, self.wte_weight, embd)
 
-        if ggml.GGML_USE_CUBLAS:
+        if ggml.GGML_USE_CUDA:
             offload_func_nr = ggml.ggml_cuda_assign_buffers_no_scratch
             offload_func_kq = ggml.ggml_cuda_assign_buffers_no_scratch
             offload_func_v = ggml.ggml_cuda_assign_buffers_no_scratch
@@ -479,7 +479,7 @@ class ReplitModel:
         for il in range(n_layer):
             offload_func = offload_nop
 
-            if ggml.GGML_USE_CUBLAS:
+            if ggml.GGML_USE_CUDA:
                 offload_func = ggml.ggml_cuda_assign_buffers_no_scratch
 
             # // lctx.use_buf(ctx0, 0)
@@ -996,7 +996,7 @@ class ReplitModel:
                 print("ctx size     =", ctx_size // (1024 * 1024), "MB")
 
             # create context
-            if ggml.GGML_USE_CUBLAS:
+            if ggml.GGML_USE_CUDA:
                 weights_buffer = CudaContextBuffer(ctx_size)
                 # ggml.ggml_cuda_set_main_device(0)
             else:
@@ -1066,7 +1066,7 @@ class ReplitModel:
                         ggml.ggml_get_data(tensor)
                     )
                 )
-                if ggml.GGML_USE_CUBLAS and name.startswith("transformer.block"):
+                if ggml.GGML_USE_CUDA and name.startswith("transformer.block"):
                     should_offload_suffix = [
                         # "norm_1.weight",
                         "attn.Wqkv.weight",
@@ -1078,7 +1078,7 @@ class ReplitModel:
                     if any(name.endswith(s) for s in should_offload_suffix):
                         tensor.contents.backend = ggml.GGML_BACKEND_GPU
                         ggml.ggml_cuda_transform_tensor(tensor.contents.data, tensor)
-                if ggml.GGML_USE_CUBLAS and name.startswith("transformer.block"):
+                if ggml.GGML_USE_CUDA and name.startswith("transformer.block"):
                     if (
                         name == "transformer.wte.weight"
                         or name == "transformer.norm_f.weight"

ggml_python-0.0.31/ggml/__init__.py

@@ -0,0 +1,3 @@
+from .ggml import *
+
+__version__ = "0.0.31"