vkserve 0.1.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,42 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .eggs/
6
+
7
+ # Build artifacts
8
+ /dist/
9
+ /build/
10
+ /site/
11
+ /public/
12
+
13
+ # uv / venv
14
+ .venv/
15
+ uv.lock.bak
16
+
17
+ # Tool caches
18
+ .pytest_cache/
19
+ .ruff_cache/
20
+ .pyright/
21
+ .mypy_cache/
22
+ .moon/cache/
23
+
24
+ # Editor
25
+ .vscode/
26
+ .idea/
27
+ *.swp
28
+
29
+ # Local session logs
30
+ logs/
31
+
32
+ # Personal, machine-specific agent overrides (AGENTS.md is tracked)
33
+ AGENTS.local.md
34
+ CLAUDE.local.md
35
+
36
+ # Claude Code: per-machine local settings + worktrees are local; .claude/settings.json IS tracked.
37
+ .claude/settings.local.json
38
+ .claude/worktrees/
39
+
40
+ # Local agent wiring: names cabildo-issues-mcp, which is not a dependency of this
41
+ # package. docs/issues/ IS tracked — the register is plain markdown anyone can read.
42
+ .mcp.json
@@ -0,0 +1,12 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented in this file.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ### Added
11
+
12
+ - Initial release.
vkserve-0.1.12/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Jorge Cardona
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,131 @@
1
+ Metadata-Version: 2.5
2
+ Name: vkserve
3
+ Version: 0.1.12
4
+ Summary: An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on compute-ml
5
+ Project-URL: Homepage, https://gitlab.com/cardona-oss/vkserve
6
+ Project-URL: Repository, https://gitlab.com/cardona-oss/vkserve
7
+ Project-URL: Documentation, https://cardona-oss.gitlab.io/vkserve/
8
+ Project-URL: Changelog, https://gitlab.com/cardona-oss/vkserve/-/blob/main/CHANGELOG.md
9
+ Project-URL: Issues, https://gitlab.com/cardona-oss/vkserve/-/issues
10
+ Author-email: Jorge Cardona <jorgeecardona@gmail.com>
11
+ License-Expression: MIT
12
+ License-File: LICENSE
13
+ Classifier: Development Status :: 2 - Pre-Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Typing :: Typed
20
+ Requires-Python: >=3.12
21
+ Requires-Dist: huggingface-hub>=0.26
22
+ Requires-Dist: jinja2>=3.1
23
+ Requires-Dist: litestar>=2.12
24
+ Requires-Dist: tokenizers>=0.20
25
+ Requires-Dist: uvicorn>=0.30
26
+ Provides-Extra: docs
27
+ Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
28
+ Requires-Dist: mkdocs>=1.6; extra == 'docs'
29
+ Requires-Dist: mkdocstrings[python]>=0.27; extra == 'docs'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # vkserve
33
+
34
+ An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on
35
+ [compute-ml](https://gitlab.com/cardona-oss/compute-ml). Status: prototype, not released.
36
+
37
+ ## Run it
38
+
39
+ 1. Build the C engine. It compiles compute-ml from `../compute-ml`.
40
+
41
+ ```bash
42
+ make engine
43
+ ```
44
+
45
+ 2. Start the server on a model folder or a Hugging Face id. The id is downloaded on first use.
46
+
47
+ ```bash
48
+ uv run vkserve Qwen/Qwen3-0.6B --dtype q4 --slots 1 --max-len 3000
49
+ ```
50
+
51
+ The model id in the API is the last part of the name (`Qwen3-0.6B`); `--name` changes it.
52
+
53
+ Pick the GPU with `--device` (an index or part of the name). `vkserve --list-devices` shows
54
+ them. Repeat `--device` to run the model on several GPUs: requests go to the least busy one,
55
+ and a GPU that cannot load the model is skipped.
56
+
57
+ ```bash
58
+ uv run vkserve --list-devices
59
+ uv run vkserve Qwen/Qwen3-0.6B --device intel --device t2000
60
+ ```
61
+
62
+ Models: Qwen3, Llama 3.2, SmolLM2, Gemma 3 and Gemma 4, as safetensors with a
63
+ `tokenizer.json`. A bare `.gguf` file is not supported yet. Several models in one server:
64
+ repeat `--model name=ID,path=FOLDER_OR_HF_ID[,dtype=q8][,slots=4][,max_len=4096]`.
65
+
66
+ 3. Send a request.
67
+
68
+ ```bash
69
+ curl -s localhost:8000/v1/chat/completions -H 'content-type: application/json' \
70
+ -d '{"messages":[{"role":"user","content":"Name two planets."}],"max_tokens":30}'
71
+ ```
72
+
73
+ ## Use it from opencode
74
+
75
+ See [examples/opencode](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/opencode/README.md): a config, a small agent, and the commands.
76
+
77
+ ## Configuration
78
+
79
+ One TOML file, and every key is also a flag. Flags win over the file. See
80
+ [examples/vkserve.toml](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.toml).
81
+
82
+ ```bash
83
+ uv run vkserve -t -c vkserve.toml # check the file, load nothing
84
+ uv run vkserve -c vkserve.toml # run it
85
+ uv run vkserve -c vkserve.toml --port 9000 # a flag overrides the file
86
+ uv run vkserve -c vkserve.toml --model qwen --slots 2 # change one model of the file
87
+ uv run vkserve --print-config -c vkserve.toml # show every effective setting
88
+ ```
89
+
90
+ - `[server]` keys are plain flags: `--host`, `--port`, `--log-level`, `--keep-going`,
91
+ `--max-body-bytes`, `--pid-file`, `--log-file`, `--drain-timeout`.
92
+ - `--model NAME` starts a model block. The flags after it apply to that block: `--path`,
93
+ `--dtype`, `--slots`, `--max-len`, `--prefill-chunk`, `--max-queue`, `--device` (repeat for
94
+ several devices), `--temperature`, `--top-p`, `--top-k`.
95
+ - `--set server.port=9000` and `--set model.qwen.slots=2` reach any key.
96
+ - An unknown key is an error with a hint: `unknown key 'slot' ... (did you mean 'slots'?)`.
97
+
98
+ ## Run it as a service
99
+
100
+ The master process serves HTTP. Each model on each device runs in its own worker process. A
101
+ worker that crashes fails its requests with 500 and starts again after 1, 2, 4 ... 30 seconds.
102
+ The master keeps running.
103
+
104
+ Send signals with `kill`, or with `-s` (it reads `pid_file`):
105
+
106
+ | Signal | `-s` | What it does |
107
+ |---|---|---|
108
+ | `SIGHUP` | `reload` | Read the configuration again. A new or changed model loads first, then the old one finishes its requests and stops. A bad file changes nothing. |
109
+ | `SIGTERM`, `SIGQUIT`, `SIGINT` | `stop`, `quit` | Stop accepting, wait up to `drain_timeout` for requests in progress, then exit. A second signal stops at once. |
110
+ | `SIGUSR1` | `reopen` | Reopen `log_file` (after logrotate moved it). |
111
+
112
+ `host`, `port`, `max_body_bytes`, `pid_file`, `drain_timeout` and `keep_going` need a restart.
113
+ A reload that changes them logs a warning and ignores them.
114
+
115
+ ```bash
116
+ uv run vkserve -s reload -c vkserve.toml
117
+ ```
118
+
119
+ A systemd unit is in [examples/vkserve.service](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.service).
120
+
121
+ ## Features
122
+
123
+ - `POST /v1/chat/completions` and `/v1/completions`, streaming as server-sent events.
124
+ - Tool calls: `tools` in, OpenAI `tool_calls` out, `tool` messages back in.
125
+ - Several requests in one batched forward pass per step.
126
+ - Several GPUs: one worker per device, least-busy routing, failed workers skipped and restarted.
127
+ - Operations: `429` when the queue is full, `X-Request-ID`, Prometheus `/metrics` per device.
128
+ - Prefix reuse: a request that starts with a cached conversation prefills only the new tokens.
129
+ - Attention over long contexts (tested to 20,000 keys).
130
+
131
+ More: [docs/serving.md](https://gitlab.com/cardona-oss/vkserve/-/blob/main/docs/serving.md) (design and measurements).
@@ -0,0 +1,100 @@
1
+ # vkserve
2
+
3
+ An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on
4
+ [compute-ml](https://gitlab.com/cardona-oss/compute-ml). Status: prototype, not released.
5
+
6
+ ## Run it
7
+
8
+ 1. Build the C engine. It compiles compute-ml from `../compute-ml`.
9
+
10
+ ```bash
11
+ make engine
12
+ ```
13
+
14
+ 2. Start the server on a model folder or a Hugging Face id. The id is downloaded on first use.
15
+
16
+ ```bash
17
+ uv run vkserve Qwen/Qwen3-0.6B --dtype q4 --slots 1 --max-len 3000
18
+ ```
19
+
20
+ The model id in the API is the last part of the name (`Qwen3-0.6B`); `--name` changes it.
21
+
22
+ Pick the GPU with `--device` (an index or part of the name). `vkserve --list-devices` shows
23
+ them. Repeat `--device` to run the model on several GPUs: requests go to the least busy one,
24
+ and a GPU that cannot load the model is skipped.
25
+
26
+ ```bash
27
+ uv run vkserve --list-devices
28
+ uv run vkserve Qwen/Qwen3-0.6B --device intel --device t2000
29
+ ```
30
+
31
+ Models: Qwen3, Llama 3.2, SmolLM2, Gemma 3 and Gemma 4, as safetensors with a
32
+ `tokenizer.json`. A bare `.gguf` file is not supported yet. Several models in one server:
33
+ repeat `--model name=ID,path=FOLDER_OR_HF_ID[,dtype=q8][,slots=4][,max_len=4096]`.
34
+
35
+ 3. Send a request.
36
+
37
+ ```bash
38
+ curl -s localhost:8000/v1/chat/completions -H 'content-type: application/json' \
39
+ -d '{"messages":[{"role":"user","content":"Name two planets."}],"max_tokens":30}'
40
+ ```
41
+
42
+ ## Use it from opencode
43
+
44
+ See [examples/opencode](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/opencode/README.md): a config, a small agent, and the commands.
45
+
46
+ ## Configuration
47
+
48
+ One TOML file, and every key is also a flag. Flags win over the file. See
49
+ [examples/vkserve.toml](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.toml).
50
+
51
+ ```bash
52
+ uv run vkserve -t -c vkserve.toml # check the file, load nothing
53
+ uv run vkserve -c vkserve.toml # run it
54
+ uv run vkserve -c vkserve.toml --port 9000 # a flag overrides the file
55
+ uv run vkserve -c vkserve.toml --model qwen --slots 2 # change one model of the file
56
+ uv run vkserve --print-config -c vkserve.toml # show every effective setting
57
+ ```
58
+
59
+ - `[server]` keys are plain flags: `--host`, `--port`, `--log-level`, `--keep-going`,
60
+ `--max-body-bytes`, `--pid-file`, `--log-file`, `--drain-timeout`.
61
+ - `--model NAME` starts a model block. The flags after it apply to that block: `--path`,
62
+ `--dtype`, `--slots`, `--max-len`, `--prefill-chunk`, `--max-queue`, `--device` (repeat for
63
+ several devices), `--temperature`, `--top-p`, `--top-k`.
64
+ - `--set server.port=9000` and `--set model.qwen.slots=2` reach any key.
65
+ - An unknown key is an error with a hint: `unknown key 'slot' ... (did you mean 'slots'?)`.
66
+
67
+ ## Run it as a service
68
+
69
+ The master process serves HTTP. Each model on each device runs in its own worker process. A
70
+ worker that crashes fails its requests with 500 and starts again after 1, 2, 4 ... 30 seconds.
71
+ The master keeps running.
72
+
73
+ Send signals with `kill`, or with `-s` (it reads `pid_file`):
74
+
75
+ | Signal | `-s` | What it does |
76
+ |---|---|---|
77
+ | `SIGHUP` | `reload` | Read the configuration again. A new or changed model loads first, then the old one finishes its requests and stops. A bad file changes nothing. |
78
+ | `SIGTERM`, `SIGQUIT`, `SIGINT` | `stop`, `quit` | Stop accepting, wait up to `drain_timeout` for requests in progress, then exit. A second signal stops at once. |
79
+ | `SIGUSR1` | `reopen` | Reopen `log_file` (after logrotate moved it). |
80
+
81
+ `host`, `port`, `max_body_bytes`, `pid_file`, `drain_timeout` and `keep_going` need a restart.
82
+ A reload that changes them logs a warning and ignores them.
83
+
84
+ ```bash
85
+ uv run vkserve -s reload -c vkserve.toml
86
+ ```
87
+
88
+ A systemd unit is in [examples/vkserve.service](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.service).
89
+
90
+ ## Features
91
+
92
+ - `POST /v1/chat/completions` and `/v1/completions`, streaming as server-sent events.
93
+ - Tool calls: `tools` in, OpenAI `tool_calls` out, `tool` messages back in.
94
+ - Several requests in one batched forward pass per step.
95
+ - Several GPUs: one worker per device, least-busy routing, failed workers skipped and restarted.
96
+ - Operations: `429` when the queue is full, `X-Request-ID`, Prometheus `/metrics` per device.
97
+ - Prefix reuse: a request that starts with a cached conversation prefills only the new tokens.
98
+ - Attention over long contexts (tested to 20,000 keys).
99
+
100
+ More: [docs/serving.md](https://gitlab.com/cardona-oss/vkserve/-/blob/main/docs/serving.md) (design and measurements).
@@ -0,0 +1,25 @@
1
+ # libvkserve_engine: the multi-sequence engine, loaded by src/vkserve/engine.py through ctypes.
2
+ # It builds compute-ml from source (COMPUTE_ML_DIR, default ../compute-ml next to this repo) and
3
+ # compiles compute-ml's LLM core (examples/llm) into the same library.
4
+ cmake_minimum_required(VERSION 3.21)
5
+ project(vkserve_engine LANGUAGES C)
6
+ set(CMAKE_C_STANDARD 11)
7
+ set(CMAKE_C_STANDARD_REQUIRED ON)
8
+
9
+ set(COMPUTE_ML_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../compute-ml" CACHE PATH "compute-ml source tree")
10
+ get_filename_component(COMPUTE_ML_DIR "${COMPUTE_ML_DIR}" ABSOLUTE)
11
+ if(NOT EXISTS "${COMPUTE_ML_DIR}/examples/llm/llm.c")
12
+ message(FATAL_ERROR "compute-ml not found at ${COMPUTE_ML_DIR}; pass -DCOMPUTE_ML_DIR=<path>")
13
+ endif()
14
+ add_subdirectory("${COMPUTE_ML_DIR}" compute-ml EXCLUDE_FROM_ALL)
15
+ find_package(Vulkan REQUIRED)
16
+
17
+ add_library(vkserve_engine SHARED engine.c devices.c
18
+ "${COMPUTE_ML_DIR}/examples/llm/llm.c" "${COMPUTE_ML_DIR}/examples/whisper/json.c")
19
+ target_include_directories(vkserve_engine PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
20
+ "${COMPUTE_ML_DIR}/examples/llm" "${COMPUTE_ML_DIR}/examples/whisper")
21
+ target_link_libraries(vkserve_engine PRIVATE compute_ml Vulkan::Vulkan)
22
+ if(NOT MSVC)
23
+ target_link_libraries(vkserve_engine PRIVATE m)
24
+ endif()
25
+ set_target_properties(vkserve_engine PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
@@ -0,0 +1,60 @@
1
+ /* List the Vulkan devices that can run compute, in the order compute-ml numbers them
2
+ * (the index CML_GPU_INDEX takes). */
3
+
4
+ #include <stdio.h>
5
+ #include <stdlib.h>
6
+ #include <string.h>
7
+
8
+ #include <vulkan/vulkan.h>
9
+
10
+ static const char *kind_name(VkPhysicalDeviceType t) {
11
+ switch (t) {
12
+ case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
13
+ case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
14
+ case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
15
+ case VK_PHYSICAL_DEVICE_TYPE_CPU: return "cpu";
16
+ default: return "other";
17
+ }
18
+ }
19
+
20
+ static int has_compute(VkPhysicalDevice d) {
21
+ uint32_t n = 0;
22
+ vkGetPhysicalDeviceQueueFamilyProperties(d, &n, NULL);
23
+ VkQueueFamilyProperties *q = malloc((n ? n : 1) * sizeof *q);
24
+ if (!q) return 0;
25
+ vkGetPhysicalDeviceQueueFamilyProperties(d, &n, q);
26
+ int ok = 0;
27
+ for (uint32_t i = 0; i < n && !ok; i++) ok = (q[i].queueFlags & VK_QUEUE_COMPUTE_BIT) != 0;
28
+ free(q);
29
+ return ok;
30
+ }
31
+
32
+ /* Write one line per device into buf: "<index>\t<kind>\t<compute 0|1>\t<name>\n". Returns the
33
+ * number of devices, or -1 when Vulkan cannot start. buf is always NUL-terminated. */
34
+ int vkserve_devices(char *buf, size_t cap) {
35
+ if (!buf || !cap) return -1;
36
+ buf[0] = 0;
37
+ VkApplicationInfo app = { .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
38
+ .apiVersion = VK_API_VERSION_1_1 };
39
+ VkInstanceCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
40
+ .pApplicationInfo = &app };
41
+ VkInstance inst;
42
+ if (vkCreateInstance(&ci, NULL, &inst) != VK_SUCCESS) return -1;
43
+ uint32_t n = 0;
44
+ vkEnumeratePhysicalDevices(inst, &n, NULL);
45
+ VkPhysicalDevice *devs = malloc((n ? n : 1) * sizeof *devs);
46
+ if (!devs) { vkDestroyInstance(inst, NULL); return -1; }
47
+ vkEnumeratePhysicalDevices(inst, &n, devs);
48
+ size_t used = 0;
49
+ for (uint32_t i = 0; i < n; i++) {
50
+ VkPhysicalDeviceProperties p;
51
+ vkGetPhysicalDeviceProperties(devs[i], &p);
52
+ int w = snprintf(buf + used, cap - used, "%u\t%s\t%d\t%s\n", i, kind_name(p.deviceType),
53
+ has_compute(devs[i]), p.deviceName);
54
+ if (w < 0 || (size_t)w >= cap - used) break;
55
+ used += (size_t)w;
56
+ }
57
+ free(devs);
58
+ vkDestroyInstance(inst, NULL);
59
+ return (int)n;
60
+ }