aikernel-cuda-driver 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. aikernel_cuda_driver-0.1.4/CMakeLists.txt +42 -0
  2. aikernel_cuda_driver-0.1.4/LICENSE +189 -0
  3. aikernel_cuda_driver-0.1.4/MANIFEST.in +4 -0
  4. aikernel_cuda_driver-0.1.4/PKG-INFO +129 -0
  5. aikernel_cuda_driver-0.1.4/README.md +106 -0
  6. aikernel_cuda_driver-0.1.4/build_gemma_ptx.py +29 -0
  7. aikernel_cuda_driver-0.1.4/pyproject.toml +35 -0
  8. aikernel_cuda_driver-0.1.4/python/aikernel_cuda_driver/__init__.py +161 -0
  9. aikernel_cuda_driver-0.1.4/python/aikernel_cuda_driver.egg-info/PKG-INFO +129 -0
  10. aikernel_cuda_driver-0.1.4/python/aikernel_cuda_driver.egg-info/SOURCES.txt +31 -0
  11. aikernel_cuda_driver-0.1.4/python/aikernel_cuda_driver.egg-info/dependency_links.txt +1 -0
  12. aikernel_cuda_driver-0.1.4/python/aikernel_cuda_driver.egg-info/top_level.txt +1 -0
  13. aikernel_cuda_driver-0.1.4/setup.cfg +4 -0
  14. aikernel_cuda_driver-0.1.4/setup.py +42 -0
  15. aikernel_cuda_driver-0.1.4/src/aikernel_cuda.c +687 -0
  16. aikernel_cuda_driver-0.1.4/src/aikernel_cuda.h +101 -0
  17. aikernel_cuda_driver-0.1.4/src/cpu_dot_avx.c +19 -0
  18. aikernel_cuda_driver-0.1.4/src/cpu_dot_avx2.c +19 -0
  19. aikernel_cuda_driver-0.1.4/src/cpu_dot_avx512.c +19 -0
  20. aikernel_cuda_driver-0.1.4/src/cpu_gemv.c +80 -0
  21. aikernel_cuda_driver-0.1.4/src/cpu_gemv.h +8 -0
  22. aikernel_cuda_driver-0.1.4/src/driver.c +77 -0
  23. aikernel_cuda_driver-0.1.4/src/driver.h +49 -0
  24. aikernel_cuda_driver-0.1.4/src/embedded_int8.h.in +1 -0
  25. aikernel_cuda_driver-0.1.4/src/embedded_ptx.h.in +1 -0
  26. aikernel_cuda_driver-0.1.4/src/gemma_forward.cu +69 -0
  27. aikernel_cuda_driver-0.1.4/src/gemma_forward.ptx +1709 -0
  28. aikernel_cuda_driver-0.1.4/src/gemma_gemv.cu +29 -0
  29. aikernel_cuda_driver-0.1.4/src/gemma_gemv.ptx +224 -0
  30. aikernel_cuda_driver-0.1.4/src/gemma_program.inc +122 -0
  31. aikernel_cuda_driver-0.1.4/src/kernels.ptx +371 -0
  32. aikernel_cuda_driver-0.1.4/src/q1_dp4a.ptx +48 -0
  33. aikernel_cuda_driver-0.1.4/src/q1_mma.ptx +63 -0
@@ -0,0 +1,42 @@
1
+ cmake_minimum_required(VERSION 3.24)
2
+ project(aikernel_cuda_driver LANGUAGES C)
3
+
4
+ if(NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64)$")
5
+ message(FATAL_ERROR "The CUDA Driver ABI is built for Windows or Linux x86-64.")
6
+ endif()
7
+ if(NOT WIN32 AND NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
8
+ message(FATAL_ERROR "The CUDA Driver ABI is built for Windows or Linux.")
9
+ endif()
10
+
11
+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS src/kernels.ptx)
12
+ file(READ "${CMAKE_CURRENT_SOURCE_DIR}/src/kernels.ptx" PTX_LITERAL)
13
+ string(REPLACE "\\" "\\\\" PTX_LITERAL "${PTX_LITERAL}")
14
+ string(REPLACE "\"" "\\\"" PTX_LITERAL "${PTX_LITERAL}")
15
+ string(REPLACE "\n" "\\n\"\n\"" PTX_LITERAL "${PTX_LITERAL}")
16
+ configure_file(src/embedded_ptx.h.in embedded_ptx.h @ONLY)
17
+
18
+ foreach(KERNEL q1_dp4a q1_mma gemma_gemv gemma_forward)
19
+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS src/${KERNEL}.ptx)
20
+ file(READ "${CMAKE_CURRENT_SOURCE_DIR}/src/${KERNEL}.ptx" PTX_LITERAL)
21
+ string(REPLACE "\\" "\\\\" PTX_LITERAL "${PTX_LITERAL}")
22
+ string(REPLACE "\"" "\\\"" PTX_LITERAL "${PTX_LITERAL}")
23
+ string(REPLACE "\n" "\\n\"\n\"" PTX_LITERAL "${PTX_LITERAL}")
24
+ configure_file(src/embedded_int8.h.in ${KERNEL}.h @ONLY)
25
+ endforeach()
26
+
27
+ add_library(aikernel_cuda_driver SHARED src/aikernel_cuda.c src/driver.c src/cpu_gemv.c
28
+ src/cpu_dot_avx.c src/cpu_dot_avx2.c src/cpu_dot_avx512.c)
29
+ target_include_directories(aikernel_cuda_driver PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
30
+ set_target_properties(aikernel_cuda_driver PROPERTIES C_STANDARD 11 C_STANDARD_REQUIRED YES
31
+ C_VISIBILITY_PRESET hidden MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
32
+ if(MSVC)
33
+ target_compile_options(aikernel_cuda_driver PRIVATE /W4 /WX)
34
+ # Never apply /arch to the dispatcher or the whole DLL.
35
+ set_source_files_properties(src/cpu_dot_avx.c PROPERTIES COMPILE_OPTIONS /arch:AVX)
36
+ set_source_files_properties(src/cpu_dot_avx2.c PROPERTIES COMPILE_OPTIONS /arch:AVX2)
37
+ set_source_files_properties(src/cpu_dot_avx512.c PROPERTIES COMPILE_OPTIONS /arch:AVX512)
38
+ else()
39
+ target_compile_options(aikernel_cuda_driver PRIVATE -Wall -Wextra -Werror)
40
+ target_link_libraries(aikernel_cuda_driver PRIVATE ${CMAKE_DL_LIBS} m)
41
+ endif()
42
+ install(TARGETS aikernel_cuda_driver RUNTIME DESTINATION aikernel_cuda_driver LIBRARY DESTINATION aikernel_cuda_driver)
@@ -0,0 +1,189 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ https://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ patent license to make, have made, use, offer to sell, sell, import,
77
+ and otherwise transfer the Work, where such license applies only to
78
+ those patent claims licensable by such Contributor that are necessarily
79
+ infringed by their Contribution(s) alone or by combination of their
80
+ Contribution(s) with the Work to which such Contribution(s) was
81
+ submitted. If You institute patent litigation against any entity
82
+ (including a cross-claim or counterclaim in a lawsuit) alleging that
83
+ the Work or a Contribution incorporated within the Work constitutes
84
+ direct or contributory patent infringement, then any patent licenses
85
+ granted to You under this License for that Work shall terminate as of
86
+ the date such litigation is filed.
87
+
88
+ 4. Redistribution. You may reproduce and distribute copies of the
89
+ Work or Derivative Works thereof in any medium, with or without
90
+ modifications, and in Source or Object form, provided that You
91
+ meet the following conditions:
92
+
93
+ (a) You must give any other recipients of the Work or
94
+ Derivative Works a copy of this License; and
95
+
96
+ (b) You must cause any modified files to carry prominent notices
97
+ stating that You changed the files; and
98
+
99
+ (c) You must retain, in the Source form of any Derivative Works
100
+ that You distribute, all copyright, patent, trademark, and
101
+ attribution notices from the Source form of the Work,
102
+ excluding those notices that do not pertain to any part of
103
+ the Derivative Works; and
104
+
105
+ (d) If the Work includes a "NOTICE" text file as part of its
106
+ distribution, then any Derivative Works that You distribute must
107
+ include a readable copy of the attribution notices contained
108
+ within such NOTICE file, excluding those notices that do not
109
+ pertain to any part of the Derivative Works, in at least one
110
+ of the following places: within a NOTICE text file distributed
111
+ as part of the Derivative Works; within the Source form or
112
+ documentation, if provided along with the Derivative Works; or,
113
+ within a display generated by the Derivative Works, if and
114
+ wherever such third-party notices normally appear. The contents
115
+ of the NOTICE file are for informational purposes only and
116
+ do not modify the License. You may add Your own attribution
117
+ notices within Derivative Works that You distribute, alongside
118
+ or as an addendum to the NOTICE text from the Work, provided
119
+ that such additional attribution notices cannot be construed
120
+ as modifying the License.
121
+
122
+ You may add Your own copyright statement to Your modifications and
123
+ may provide additional or different license terms and conditions
124
+ for use, reproduction, or distribution of Your modifications, or
125
+ for any such Derivative Works as a whole, provided Your use,
126
+ reproduction, and distribution of the Work otherwise complies with
127
+ the conditions stated in this License.
128
+
129
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
130
+ any Contribution intentionally submitted for inclusion in the Work
131
+ by You to the Licensor shall be under the terms and conditions of
132
+ this License, without any additional terms or conditions.
133
+ Notwithstanding the above, nothing herein shall supersede or modify
134
+ the terms of any separate license agreement you may have executed
135
+ with Licensor regarding such Contributions.
136
+
137
+ 6. Trademarks. This License does not grant permission to use the trade
138
+ names, trademarks, service marks, or product names of the Licensor,
139
+ except as required for reasonable and customary use in describing the
140
+ origin of the Work and reproducing the content of the NOTICE file.
141
+
142
+ 7. Disclaimer of Warranty. Unless required by applicable law or
143
+ agreed to in writing, Licensor provides the Work (and each
144
+ Contributor provides its Contributions) on an "AS IS" BASIS,
145
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
146
+ implied, including, without limitation, any warranties or conditions
147
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
148
+ PARTICULAR PURPOSE. You are solely responsible for determining the
149
+ appropriateness of using or redistributing the Work and assume any
150
+ risks associated with Your exercise of permissions under this License.
151
+
152
+ 8. Limitation of Liability. In no event and under no legal theory,
153
+ whether in tort (including negligence), contract, or otherwise,
154
+ unless required by applicable law (such as deliberate and grossly
155
+ negligent acts) or agreed to in writing, shall any Contributor be
156
+ liable to You for damages, including any direct, indirect, special,
157
+ incidental, or consequential damages of any character arising as a
158
+ result of this License or out of the use or inability to use the
159
+ Work (including but not limited to damages for loss of goodwill,
160
+ work stoppage, computer failure or malfunction, or any and all
161
+ other commercial damages or losses), even if such Contributor
162
+ has been advised of the possibility of such damages.
163
+
164
+ 9. Accepting Warranty or Additional Liability. While redistributing
165
+ the Work or Derivative Works thereof, You may choose to offer,
166
+ and charge a fee for, acceptance of support, warranty, indemnity,
167
+ or other liability obligations and/or rights consistent with this
168
+ License. However, in accepting such obligations, You may act only
169
+ on Your own behalf and on Your sole responsibility, not on behalf
170
+ of any other Contributor, and only if You agree to indemnify,
171
+ defend, and hold each Contributor harmless for any liability
172
+ incurred by, or claims asserted against, such Contributor by reason
173
+ of your accepting any such warranty or additional liability.
174
+
175
+ END OF TERMS AND CONDITIONS
176
+
177
+ Copyright 2026 Takuya Sogawa of AIKernel-NET
178
+
179
+ Licensed under the Apache License, Version 2.0 (the "License");
180
+ you may not use this file except in compliance with the License.
181
+ You may obtain a copy of the License at
182
+
183
+ https://www.apache.org/licenses/LICENSE-2.0
184
+
185
+ Unless required by applicable law or agreed to in writing, software
186
+ distributed under the License is distributed on an "AS IS" BASIS,
187
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
188
+ See the License for the specific language governing permissions and
189
+ limitations under the License.
@@ -0,0 +1,4 @@
1
+ include LICENSE
2
+ include CMakeLists.txt
3
+ recursive-include src *.c *.h *.in *.inc *.cu *.ptx
4
+ include build_gemma_ptx.py
@@ -0,0 +1,129 @@
1
+ Metadata-Version: 2.4
2
+ Name: aikernel-cuda-driver
3
+ Version: 0.1.4
4
+ Summary: AIKernel Native C ABI with a dynamically loaded CUDA Driver backend
5
+ Author: Takuya Sogawa
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/AIKernel-NET/AIKernel.Core
8
+ Project-URL: Repository, https://github.com/AIKernel-NET/AIKernel.Core
9
+ Project-URL: Issues, https://github.com/AIKernel-NET/AIKernel.Core/issues
10
+ Keywords: aikernel,cuda,cuda-driver,ctypes,native-abi
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: Microsoft :: Windows
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: C
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Dynamic: license-file
23
+
24
+ # AIKernel CUDA Driver
25
+
26
+ AIKernel Native C ABI for Windows and Linux x86-64. Build with a C compiler
27
+ and CMake. The runtime loads NVIDIA's installed driver on demand. Python
28
+ uses ctypes to run the same primitives on CUDA or the host.
29
+
30
+ Build a Windows wheel with python -m build --wheel. Build and repair Linux
31
+ wheels in the manylinux_2_28_x86_64 image using auditwheel.
32
+
33
+ The C ABI uses caller-owned buffers. CUDA allocations belong to one private
34
+ context and calls on that context must be serialized. Transfers use page-locked
35
+ host buffers and complete before the ABI call returns. Context destruction
36
+ reclaims its device allocations, stream and module. See src/aikernel_cuda.h for
37
+ the ABI version, status codes and counters.
38
+
39
+ ## CPU instruction dispatch
40
+
41
+ Host FP32 GEMV selects AVX-512, AVX2+FMA, AVX, or scalar at runtime. CPU and
42
+ OS XSAVE support are checked before calling an ISA-specific function. CUDA
43
+ selection remains independent. Keep the dispatcher at the baseline x86-64
44
+ ISA; never build the whole library with `-march=native` or `/arch:AVX512`.
45
+ CMake assigns MSVC `/arch` flags only to the three dedicated source files.
46
+ GCC/Clang use per-function target attributes. An AVX-only CPU does not need FMA.
47
+
48
+ `AIKERNEL_CPU_GEMV=auto` is the default. Validation can force `scalar`, `avx`,
49
+ `avx2`, or `avx512`. An unsupported or unknown forced value makes host GEMV
50
+ return `AK_INVALID` without touching the output. `ak_cpu_gemv_backend()` reports
51
+ the selected enum or `AK_CPU_UNAVAILABLE`. Set the environment before inference;
52
+ do not change process environment concurrently with calls.
53
+
54
+ Run `python tests/cpu_dispatch.py /absolute/path/to/library --benchmark` to
55
+ compare against a double-precision reference, exercise unaligned input/tails,
56
+ and verify unsupported-path rejection. The optional single-thread benchmark
57
+ uses a 256 x 2048 FP32 matrix; it does not measure LLM tokens/second. These
58
+ host CPU changes do not add VNNI or BF16 kernels.
59
+
60
+ Linux packages for XServer require glibc 2.28 compatibility. Build in the
61
+ manylinux_2_28 environment, then test that exact library on both AVX2 and
62
+ AVX-512 hosts. Do not distribute a library built against the development
63
+ machine's newer glibc without checking its symbol requirements.
64
+
65
+ ## Quantized Qwen3 matrix offload
66
+
67
+ The managed `Qwen3GpuModel` runs the same bounded dense Qwen3/Bonsai transformer
68
+ as `Qwen3CpuModel`, with all matrix-vector products on the selected CUDA device.
69
+ Embedding lookup, normalization, RoPE, attention, KV storage and sampling remain
70
+ on the CPU. This is a hybrid execution path, not an all-GPU transformer.
71
+ GPU errors propagate; no matrix operation silently switches to the CPU.
72
+
73
+ ~~~csharp
74
+ using var model = new AIKernel.Kernel.Inference.Qwen3GpuModel(
75
+ modelPath, deviceOrdinal: 0, cpuThreads: 4, contextTokens: 1024,
76
+ maximumWeightBytes: 1024L * 1024 * 1024);
77
+ await model.WarmupAsync(systemInstructions, cancellationToken);
78
+ var reply = await model.GenerateAsync(systemInstructions, inquiry, 96, cancellationToken);
79
+ Console.WriteLine(reply.Text);
80
+ ~~~
81
+
82
+ Deploy the native library from this source revision alongside the managed assembly.
83
+ The official `AIKernel.Kernel` NuGet release includes matching `win-x64` and
84
+ `linux-x64` runtime assets, so normal RID-aware restore/publish selects the library.
85
+ Release packaging requires both assets; a source-only CPU build may omit them.
86
+ NVIDIA's installed driver is required; a CUDA Toolkit or LibTorch installation is
87
+ not required. CPU-only callers still use `Qwen3CpuModel` without loading this library.
88
+ An invalid device, missing native export, unsupported model or exhausted allocation
89
+ fails the request. Dispose the model after its active requests have stopped.
90
+
91
+ The device holds the encoded GGUF file within `maximumWeightBytes`. The budget
92
+ covers model weights, not total GPU usage: the driver context, matrix working
93
+ buffers and up to one MiB of upload staging are additional. Leave headroom for
94
+ other GPU users. Uploads are chunked; the whole model is never expanded to FP32.
95
+ Only supported dense Qwen3 layouts with FP32 or GGUF type 41 matrices are accepted;
96
+ this does not imply support for all models below four billion parameters.
97
+
98
+ The additive `ak_gemv_q1_0` export takes device-resident Q1_0 g128 blocks:
99
+ a little-endian FP16 scale followed by 16 sign bytes per 128 values. A zero bit
100
+ represents minus scale; a set bit represents plus scale. Columns must be a multiple
101
+ of 128 and the address must be aligned to two bytes. The export requires a CUDA
102
+ context. FP32 and quantized device GEMV accept bounded interior tensor ranges.
103
+ Transfers also accept interior ranges, while free requires the original allocation
104
+ base. Bounds are checked against allocations owned by the context before launch.
105
+
106
+ Run independent arithmetic and allocation-boundary checks on a real device:
107
+
108
+ ~~~sh
109
+ python tests/q1_gpu.py /absolute/path/to/library --device 0
110
+ ~~~
111
+
112
+ Run managed end-to-end validation with a local licensed model:
113
+
114
+ ~~~sh
115
+ dotnet run -c Release --project ../../tests/AIKernel.GpuInference.Validation -- /path/to/Bonsai-4B-Q1_0.gguf
116
+ ~~~
117
+
118
+ That harness compares CPU and GPU text for one fixed Japanese support inquiry,
119
+ checks the encoded-weight budget, serialization, cancellation and recovery.
120
+ Its timings are functional diagnostics, not controlled performance measurements.
121
+ Equal output on these inputs is regression evidence, not proof of universal
122
+ numerical equivalence. Benchmark CPU/GPU only after controlling background load,
123
+ with the same model hash, inputs, thread ceiling and warmup policy.
124
+
125
+ Python users install `aikernel-net[gpu]==0.1.4` and the .NET 10 runtime. Import
126
+ `aikernel_cuda_driver` before calling the managed GPU API via pythonnet, so the
127
+ matching native library from its platform wheel is loaded. The managed wrapper
128
+ wheel remains platform-independent; the optional driver wheel is available for
129
+ Windows x64 and manylinux 2.28+ x64. CPU-only imports do not require this extra.
@@ -0,0 +1,106 @@
1
+ # AIKernel CUDA Driver
2
+
3
+ AIKernel Native C ABI for Windows and Linux x86-64. Build with a C compiler
4
+ and CMake. The runtime loads NVIDIA's installed driver on demand. Python
5
+ uses ctypes to run the same primitives on CUDA or the host.
6
+
7
+ Build a Windows wheel with python -m build --wheel. Build and repair Linux
8
+ wheels in the manylinux_2_28_x86_64 image using auditwheel.
9
+
10
+ The C ABI uses caller-owned buffers. CUDA allocations belong to one private
11
+ context and calls on that context must be serialized. Transfers use page-locked
12
+ host buffers and complete before the ABI call returns. Context destruction
13
+ reclaims its device allocations, stream and module. See src/aikernel_cuda.h for
14
+ the ABI version, status codes and counters.
15
+
16
+ ## CPU instruction dispatch
17
+
18
+ Host FP32 GEMV selects AVX-512, AVX2+FMA, AVX, or scalar at runtime. CPU and
19
+ OS XSAVE support are checked before calling an ISA-specific function. CUDA
20
+ selection remains independent. Keep the dispatcher at the baseline x86-64
21
+ ISA; never build the whole library with `-march=native` or `/arch:AVX512`.
22
+ CMake assigns MSVC `/arch` flags only to the three dedicated source files.
23
+ GCC/Clang use per-function target attributes. An AVX-only CPU does not need FMA.
24
+
25
+ `AIKERNEL_CPU_GEMV=auto` is the default. Validation can force `scalar`, `avx`,
26
+ `avx2`, or `avx512`. An unsupported or unknown forced value makes host GEMV
27
+ return `AK_INVALID` without touching the output. `ak_cpu_gemv_backend()` reports
28
+ the selected enum or `AK_CPU_UNAVAILABLE`. Set the environment before inference;
29
+ do not change process environment concurrently with calls.
30
+
31
+ Run `python tests/cpu_dispatch.py /absolute/path/to/library --benchmark` to
32
+ compare against a double-precision reference, exercise unaligned input/tails,
33
+ and verify unsupported-path rejection. The optional single-thread benchmark
34
+ uses a 256 x 2048 FP32 matrix; it does not measure LLM tokens/second. These
35
+ host CPU changes do not add VNNI or BF16 kernels.
36
+
37
+ Linux packages for XServer require glibc 2.28 compatibility. Build in the
38
+ manylinux_2_28 environment, then test that exact library on both AVX2 and
39
+ AVX-512 hosts. Do not distribute a library built against the development
40
+ machine's newer glibc without checking its symbol requirements.
41
+
42
+ ## Quantized Qwen3 matrix offload
43
+
44
+ The managed `Qwen3GpuModel` runs the same bounded dense Qwen3/Bonsai transformer
45
+ as `Qwen3CpuModel`, with all matrix-vector products on the selected CUDA device.
46
+ Embedding lookup, normalization, RoPE, attention, KV storage and sampling remain
47
+ on the CPU. This is a hybrid execution path, not an all-GPU transformer.
48
+ GPU errors propagate; no matrix operation silently switches to the CPU.
49
+
50
+ ~~~csharp
51
+ using var model = new AIKernel.Kernel.Inference.Qwen3GpuModel(
52
+ modelPath, deviceOrdinal: 0, cpuThreads: 4, contextTokens: 1024,
53
+ maximumWeightBytes: 1024L * 1024 * 1024);
54
+ await model.WarmupAsync(systemInstructions, cancellationToken);
55
+ var reply = await model.GenerateAsync(systemInstructions, inquiry, 96, cancellationToken);
56
+ Console.WriteLine(reply.Text);
57
+ ~~~
58
+
59
+ Deploy the native library from this source revision alongside the managed assembly.
60
+ The official `AIKernel.Kernel` NuGet release includes matching `win-x64` and
61
+ `linux-x64` runtime assets, so normal RID-aware restore/publish selects the library.
62
+ Release packaging requires both assets; a source-only CPU build may omit them.
63
+ NVIDIA's installed driver is required; a CUDA Toolkit or LibTorch installation is
64
+ not required. CPU-only callers still use `Qwen3CpuModel` without loading this library.
65
+ An invalid device, missing native export, unsupported model or exhausted allocation
66
+ fails the request. Dispose the model after its active requests have stopped.
67
+
68
+ The device holds the encoded GGUF file within `maximumWeightBytes`. The budget
69
+ covers model weights, not total GPU usage: the driver context, matrix working
70
+ buffers and up to one MiB of upload staging are additional. Leave headroom for
71
+ other GPU users. Uploads are chunked; the whole model is never expanded to FP32.
72
+ Only supported dense Qwen3 layouts with FP32 or GGUF type 41 matrices are accepted;
73
+ this does not imply support for all models below four billion parameters.
74
+
75
+ The additive `ak_gemv_q1_0` export takes device-resident Q1_0 g128 blocks:
76
+ a little-endian FP16 scale followed by 16 sign bytes per 128 values. A zero bit
77
+ represents minus scale; a set bit represents plus scale. Columns must be a multiple
78
+ of 128 and the address must be aligned to two bytes. The export requires a CUDA
79
+ context. FP32 and quantized device GEMV accept bounded interior tensor ranges.
80
+ Transfers also accept interior ranges, while free requires the original allocation
81
+ base. Bounds are checked against allocations owned by the context before launch.
82
+
83
+ Run independent arithmetic and allocation-boundary checks on a real device:
84
+
85
+ ~~~sh
86
+ python tests/q1_gpu.py /absolute/path/to/library --device 0
87
+ ~~~
88
+
89
+ Run managed end-to-end validation with a local licensed model:
90
+
91
+ ~~~sh
92
+ dotnet run -c Release --project ../../tests/AIKernel.GpuInference.Validation -- /path/to/Bonsai-4B-Q1_0.gguf
93
+ ~~~
94
+
95
+ That harness compares CPU and GPU text for one fixed Japanese support inquiry,
96
+ checks the encoded-weight budget, serialization, cancellation and recovery.
97
+ Its timings are functional diagnostics, not controlled performance measurements.
98
+ Equal output on these inputs is regression evidence, not proof of universal
99
+ numerical equivalence. Benchmark CPU/GPU only after controlling background load,
100
+ with the same model hash, inputs, thread ceiling and warmup policy.
101
+
102
+ Python users install `aikernel-net[gpu]==0.1.4` and the .NET 10 runtime. Import
103
+ `aikernel_cuda_driver` before calling the managed GPU API via pythonnet, so the
104
+ matching native library from its platform wheel is loaded. The managed wrapper
105
+ wheel remains platform-independent; the optional driver wheel is available for
106
+ Windows x64 and manylinux 2.28+ x64. CPU-only imports do not require this extra.
@@ -0,0 +1,29 @@
1
+ """Generate the committed PTX with NVRTC (build-time only, no host compiler/STL dependency)."""
2
+ import ctypes as c
3
+ import hashlib
4
+ import json
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ lib = c.CDLL(sys.argv[1])
9
+ source = Path(__file__).parent / 'src' / (sys.argv[2] if len(sys.argv)>2 else 'gemma_gemv.cu')
10
+ target = source.with_suffix('.ptx')
11
+ program = c.c_void_p()
12
+ lib.nvrtcCreateProgram.argtypes = [c.POINTER(c.c_void_p), c.c_char_p, c.c_char_p, c.c_int, c.c_void_p, c.c_void_p]
13
+ assert lib.nvrtcCreateProgram(c.byref(program), source.read_bytes(), source.name.encode(), 0, None, None) == 0
14
+ options = [b'--gpu-architecture=compute_75', b'--fmad=false', b'--std=c++17']
15
+ options.append(('--include-path='+str(source.parent)).encode())
16
+ lib.nvrtcCompileProgram.argtypes = [c.c_void_p, c.c_int, c.POINTER(c.c_char_p)]
17
+ status = lib.nvrtcCompileProgram(program, len(options), (c.c_char_p * len(options))(*options))
18
+ size = c.c_size_t()
19
+ lib.nvrtcGetProgramLogSize(program, c.byref(size))
20
+ log = c.create_string_buffer(size.value)
21
+ lib.nvrtcGetProgramLog(program, log)
22
+ if log.value: print(log.value.decode(), file=sys.stderr)
23
+ if status: raise RuntimeError(f'NVRTC compilation failed: {status}')
24
+ lib.nvrtcGetPTXSize(program, c.byref(size))
25
+ ptx = c.create_string_buffer(size.value)
26
+ assert lib.nvrtcGetPTX(program, ptx) == 0
27
+ target.write_bytes(ptx.value)
28
+ lib.nvrtcDestroyProgram(c.byref(program))
29
+ print(json.dumps({'source': hashlib.sha256(source.read_bytes()).hexdigest(), 'ptx': hashlib.sha256(target.read_bytes()).hexdigest(), 'options': [x.decode() for x in options]}))
@@ -0,0 +1,35 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "aikernel-cuda-driver"
7
+ version = "0.1.4"
8
+ description = "AIKernel Native C ABI with a dynamically loaded CUDA Driver backend"
9
+ requires-python = ">=3.9"
10
+ readme = "README.md"
11
+ license = "Apache-2.0"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Takuya Sogawa" }]
14
+ keywords = ["aikernel", "cuda", "cuda-driver", "ctypes", "native-abi"]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Intended Audience :: Developers",
18
+ "Operating System :: Microsoft :: Windows",
19
+ "Operating System :: POSIX :: Linux",
20
+ "Programming Language :: C",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3 :: Only",
23
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
24
+ ]
25
+
26
+ [project.urls]
27
+ Homepage = "https://github.com/AIKernel-NET/AIKernel.Core"
28
+ Repository = "https://github.com/AIKernel-NET/AIKernel.Core"
29
+ Issues = "https://github.com/AIKernel-NET/AIKernel.Core/issues"
30
+
31
+ [tool.setuptools]
32
+ package-dir = {"" = "python"}
33
+
34
+ [tool.setuptools.packages.find]
35
+ where = ["python"]