vkserve 0.1.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vkserve-0.1.12/.gitignore +42 -0
- vkserve-0.1.12/CHANGELOG.md +12 -0
- vkserve-0.1.12/LICENSE +21 -0
- vkserve-0.1.12/PKG-INFO +131 -0
- vkserve-0.1.12/README.md +100 -0
- vkserve-0.1.12/csrc/CMakeLists.txt +25 -0
- vkserve-0.1.12/csrc/devices.c +60 -0
- vkserve-0.1.12/csrc/engine.c +296 -0
- vkserve-0.1.12/csrc/engine.h +77 -0
- vkserve-0.1.12/examples/opencode/README.md +86 -0
- vkserve-0.1.12/pyproject.toml +118 -0
- vkserve-0.1.12/src/vkserve/__init__.py +3 -0
- vkserve-0.1.12/src/vkserve/api.py +347 -0
- vkserve-0.1.12/src/vkserve/app.py +269 -0
- vkserve-0.1.12/src/vkserve/bench.py +125 -0
- vkserve-0.1.12/src/vkserve/cli.py +129 -0
- vkserve-0.1.12/src/vkserve/config.py +358 -0
- vkserve-0.1.12/src/vkserve/engine.py +337 -0
- vkserve-0.1.12/src/vkserve/loader.py +134 -0
- vkserve-0.1.12/src/vkserve/master.py +338 -0
- vkserve-0.1.12/src/vkserve/middleware.py +53 -0
- vkserve-0.1.12/src/vkserve/models.py +229 -0
- vkserve-0.1.12/src/vkserve/protocol.py +152 -0
- vkserve-0.1.12/src/vkserve/py.typed +0 -0
- vkserve-0.1.12/src/vkserve/remote.py +306 -0
- vkserve-0.1.12/src/vkserve/toolcalls.py +114 -0
- vkserve-0.1.12/src/vkserve/worker.py +135 -0
- vkserve-0.1.12/tests/fake_worker.py +41 -0
- vkserve-0.1.12/tests/test_api.py +361 -0
- vkserve-0.1.12/tests/test_config.py +163 -0
- vkserve-0.1.12/tests/test_engine_gpu.py +59 -0
- vkserve-0.1.12/tests/test_import.py +5 -0
- vkserve-0.1.12/tests/test_master.py +132 -0
- vkserve-0.1.12/tests/test_remote.py +139 -0
- vkserve-0.1.12/tests/test_toolcalls.py +65 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
|
|
7
|
+
# Build artifacts
|
|
8
|
+
/dist/
|
|
9
|
+
/build/
|
|
10
|
+
/site/
|
|
11
|
+
/public/
|
|
12
|
+
|
|
13
|
+
# uv / venv
|
|
14
|
+
.venv/
|
|
15
|
+
uv.lock.bak
|
|
16
|
+
|
|
17
|
+
# Tool caches
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.ruff_cache/
|
|
20
|
+
.pyright/
|
|
21
|
+
.mypy_cache/
|
|
22
|
+
.moon/cache/
|
|
23
|
+
|
|
24
|
+
# Editor
|
|
25
|
+
.vscode/
|
|
26
|
+
.idea/
|
|
27
|
+
*.swp
|
|
28
|
+
|
|
29
|
+
# Local session logs
|
|
30
|
+
logs/
|
|
31
|
+
|
|
32
|
+
# Personal, machine-specific agent overrides (AGENTS.md is tracked)
|
|
33
|
+
AGENTS.local.md
|
|
34
|
+
CLAUDE.local.md
|
|
35
|
+
|
|
36
|
+
# Claude Code: per-machine local settings + worktrees are local; .claude/settings.json IS tracked.
|
|
37
|
+
.claude/settings.local.json
|
|
38
|
+
.claude/worktrees/
|
|
39
|
+
|
|
40
|
+
# Local agent wiring: names cabildo-issues-mcp, which is not a dependency of this
|
|
41
|
+
# package. docs/issues/ IS tracked — the register is plain markdown anyone can read.
|
|
42
|
+
.mcp.json
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Initial release.
|
vkserve-0.1.12/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jorge Cardona
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
vkserve-0.1.12/PKG-INFO
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: vkserve
|
|
3
|
+
Version: 0.1.12
|
|
4
|
+
Summary: An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on compute-ml
|
|
5
|
+
Project-URL: Homepage, https://gitlab.com/cardona-oss/vkserve
|
|
6
|
+
Project-URL: Repository, https://gitlab.com/cardona-oss/vkserve
|
|
7
|
+
Project-URL: Documentation, https://cardona-oss.gitlab.io/vkserve/
|
|
8
|
+
Project-URL: Changelog, https://gitlab.com/cardona-oss/vkserve/-/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Issues, https://gitlab.com/cardona-oss/vkserve/-/issues
|
|
10
|
+
Author-email: Jorge Cardona <jorgeecardona@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Requires-Dist: huggingface-hub>=0.26
|
|
22
|
+
Requires-Dist: jinja2>=3.1
|
|
23
|
+
Requires-Dist: litestar>=2.12
|
|
24
|
+
Requires-Dist: tokenizers>=0.20
|
|
25
|
+
Requires-Dist: uvicorn>=0.30
|
|
26
|
+
Provides-Extra: docs
|
|
27
|
+
Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
|
|
28
|
+
Requires-Dist: mkdocs>=1.6; extra == 'docs'
|
|
29
|
+
Requires-Dist: mkdocstrings[python]>=0.27; extra == 'docs'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# vkserve
|
|
33
|
+
|
|
34
|
+
An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on
|
|
35
|
+
[compute-ml](https://gitlab.com/cardona-oss/compute-ml). Status: prototype, not released.
|
|
36
|
+
|
|
37
|
+
## Run it
|
|
38
|
+
|
|
39
|
+
1. Build the C engine. It compiles compute-ml from `../compute-ml`.
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
make engine
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
2. Start the server on a model folder or a Hugging Face id. The id is downloaded on first use.
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
uv run vkserve Qwen/Qwen3-0.6B --dtype q4 --slots 1 --max-len 3000
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The model id in the API is the last part of the name (`Qwen3-0.6B`); `--name` changes it.
|
|
52
|
+
|
|
53
|
+
Pick the GPU with `--device` (an index or part of the name). `vkserve --list-devices` shows
|
|
54
|
+
them. Repeat `--device` to run the model on several GPUs: requests go to the least busy one,
|
|
55
|
+
and a GPU that cannot load the model is skipped.
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv run vkserve --list-devices
|
|
59
|
+
uv run vkserve Qwen/Qwen3-0.6B --device intel --device t2000
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Models: Qwen3, Llama 3.2, SmolLM2, Gemma 3 and Gemma 4, as safetensors with a
|
|
63
|
+
`tokenizer.json`. A bare `.gguf` file is not supported yet. Several models in one server:
|
|
64
|
+
repeat `--model name=ID,path=FOLDER_OR_HF_ID[,dtype=q8][,slots=4][,max_len=4096]`.
|
|
65
|
+
|
|
66
|
+
3. Send a request.
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
curl -s localhost:8000/v1/chat/completions -H 'content-type: application/json' \
|
|
70
|
+
-d '{"messages":[{"role":"user","content":"Name two planets."}],"max_tokens":30}'
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Use it from opencode
|
|
74
|
+
|
|
75
|
+
See [examples/opencode](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/opencode/README.md): a config, a small agent, and the commands.
|
|
76
|
+
|
|
77
|
+
## Configuration
|
|
78
|
+
|
|
79
|
+
One TOML file, and every key is also a flag. Flags win over the file. See
|
|
80
|
+
[examples/vkserve.toml](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.toml).
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
uv run vkserve -t -c vkserve.toml # check the file, load nothing
|
|
84
|
+
uv run vkserve -c vkserve.toml # run it
|
|
85
|
+
uv run vkserve -c vkserve.toml --port 9000 # a flag overrides the file
|
|
86
|
+
uv run vkserve -c vkserve.toml --model qwen --slots 2 # change one model of the file
|
|
87
|
+
uv run vkserve --print-config -c vkserve.toml # show every effective setting
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
- `[server]` keys are plain flags: `--host`, `--port`, `--log-level`, `--keep-going`,
|
|
91
|
+
`--max-body-bytes`, `--pid-file`, `--log-file`, `--drain-timeout`.
|
|
92
|
+
- `--model NAME` starts a model block. The flags after it apply to that block: `--path`,
|
|
93
|
+
`--dtype`, `--slots`, `--max-len`, `--prefill-chunk`, `--max-queue`, `--device` (repeat for
|
|
94
|
+
several devices), `--temperature`, `--top-p`, `--top-k`.
|
|
95
|
+
- `--set server.port=9000` and `--set model.qwen.slots=2` reach any key.
|
|
96
|
+
- An unknown key is an error with a hint: `unknown key 'slot' ... (did you mean 'slots'?)`.
|
|
97
|
+
|
|
98
|
+
## Run it as a service
|
|
99
|
+
|
|
100
|
+
The master process serves HTTP. Each model on each device runs in its own worker process. A
|
|
101
|
+
worker that crashes fails its requests with 500 and starts again after 1, 2, 4 ... 30 seconds.
|
|
102
|
+
The master keeps running.
|
|
103
|
+
|
|
104
|
+
Send signals with `kill`, or with `-s` (it reads `pid_file`):
|
|
105
|
+
|
|
106
|
+
| Signal | `-s` | What it does |
|
|
107
|
+
|---|---|---|
|
|
108
|
+
| `SIGHUP` | `reload` | Read the configuration again. A new or changed model loads first, then the old one finishes its requests and stops. A bad file changes nothing. |
|
|
109
|
+
| `SIGTERM`, `SIGQUIT`, `SIGINT` | `stop`, `quit` | Stop accepting, wait up to `drain_timeout` for requests in progress, then exit. A second signal stops at once. |
|
|
110
|
+
| `SIGUSR1` | `reopen` | Reopen `log_file` (after logrotate moved it). |
|
|
111
|
+
|
|
112
|
+
`host`, `port`, `max_body_bytes`, `pid_file`, `drain_timeout` and `keep_going` need a restart.
|
|
113
|
+
A reload that changes them logs a warning and ignores them.
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
uv run vkserve -s reload -c vkserve.toml
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
A systemd unit is in [examples/vkserve.service](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.service).
|
|
120
|
+
|
|
121
|
+
## Features
|
|
122
|
+
|
|
123
|
+
- `POST /v1/chat/completions` and `/v1/completions`, streaming as server-sent events.
|
|
124
|
+
- Tool calls: `tools` in, OpenAI `tool_calls` out, `tool` messages back in.
|
|
125
|
+
- Several requests in one batched forward pass per step.
|
|
126
|
+
- Several GPUs: one worker per device, least-busy routing, failed workers skipped and restarted.
|
|
127
|
+
- Operations: `429` when the queue is full, `X-Request-ID`, Prometheus `/metrics` per device.
|
|
128
|
+
- Prefix reuse: a request that starts with a cached conversation prefills only the new tokens.
|
|
129
|
+
- Attention over long contexts (tested to 20,000 keys).
|
|
130
|
+
|
|
131
|
+
More: [docs/serving.md](https://gitlab.com/cardona-oss/vkserve/-/blob/main/docs/serving.md) (design and measurements).
|
vkserve-0.1.12/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# vkserve
|
|
2
|
+
|
|
3
|
+
An OpenAI-compatible LLM server that runs several models on any Vulkan GPU, built on
|
|
4
|
+
[compute-ml](https://gitlab.com/cardona-oss/compute-ml). Status: prototype, not released.
|
|
5
|
+
|
|
6
|
+
## Run it
|
|
7
|
+
|
|
8
|
+
1. Build the C engine. It compiles compute-ml from `../compute-ml`.
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
make engine
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
2. Start the server on a model folder or a Hugging Face id. The id is downloaded on first use.
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
uv run vkserve Qwen/Qwen3-0.6B --dtype q4 --slots 1 --max-len 3000
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The model id in the API is the last part of the name (`Qwen3-0.6B`); `--name` changes it.
|
|
21
|
+
|
|
22
|
+
Pick the GPU with `--device` (an index or part of the name). `vkserve --list-devices` shows
|
|
23
|
+
them. Repeat `--device` to run the model on several GPUs: requests go to the least busy one,
|
|
24
|
+
and a GPU that cannot load the model is skipped.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
uv run vkserve --list-devices
|
|
28
|
+
uv run vkserve Qwen/Qwen3-0.6B --device intel --device t2000
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Models: Qwen3, Llama 3.2, SmolLM2, Gemma 3 and Gemma 4, as safetensors with a
|
|
32
|
+
`tokenizer.json`. A bare `.gguf` file is not supported yet. Several models in one server:
|
|
33
|
+
repeat `--model name=ID,path=FOLDER_OR_HF_ID[,dtype=q8][,slots=4][,max_len=4096]`.
|
|
34
|
+
|
|
35
|
+
3. Send a request.
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
curl -s localhost:8000/v1/chat/completions -H 'content-type: application/json' \
|
|
39
|
+
-d '{"messages":[{"role":"user","content":"Name two planets."}],"max_tokens":30}'
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Use it from opencode
|
|
43
|
+
|
|
44
|
+
See [examples/opencode](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/opencode/README.md): a config, a small agent, and the commands.
|
|
45
|
+
|
|
46
|
+
## Configuration
|
|
47
|
+
|
|
48
|
+
One TOML file, and every key is also a flag. Flags win over the file. See
|
|
49
|
+
[examples/vkserve.toml](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.toml).
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
uv run vkserve -t -c vkserve.toml # check the file, load nothing
|
|
53
|
+
uv run vkserve -c vkserve.toml # run it
|
|
54
|
+
uv run vkserve -c vkserve.toml --port 9000 # a flag overrides the file
|
|
55
|
+
uv run vkserve -c vkserve.toml --model qwen --slots 2 # change one model of the file
|
|
56
|
+
uv run vkserve --print-config -c vkserve.toml # show every effective setting
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
- `[server]` keys are plain flags: `--host`, `--port`, `--log-level`, `--keep-going`,
|
|
60
|
+
`--max-body-bytes`, `--pid-file`, `--log-file`, `--drain-timeout`.
|
|
61
|
+
- `--model NAME` starts a model block. The flags after it apply to that block: `--path`,
|
|
62
|
+
`--dtype`, `--slots`, `--max-len`, `--prefill-chunk`, `--max-queue`, `--device` (repeat for
|
|
63
|
+
several devices), `--temperature`, `--top-p`, `--top-k`.
|
|
64
|
+
- `--set server.port=9000` and `--set model.qwen.slots=2` reach any key.
|
|
65
|
+
- An unknown key is an error with a hint: `unknown key 'slot' ... (did you mean 'slots'?)`.
|
|
66
|
+
|
|
67
|
+
## Run it as a service
|
|
68
|
+
|
|
69
|
+
The master process serves HTTP. Each model on each device runs in its own worker process. A
|
|
70
|
+
worker that crashes fails its requests with 500 and starts again after 1, 2, 4 ... 30 seconds.
|
|
71
|
+
The master keeps running.
|
|
72
|
+
|
|
73
|
+
Send signals with `kill`, or with `-s` (it reads `pid_file`):
|
|
74
|
+
|
|
75
|
+
| Signal | `-s` | What it does |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| `SIGHUP` | `reload` | Read the configuration again. A new or changed model loads first, then the old one finishes its requests and stops. A bad file changes nothing. |
|
|
78
|
+
| `SIGTERM`, `SIGQUIT`, `SIGINT` | `stop`, `quit` | Stop accepting, wait up to `drain_timeout` for requests in progress, then exit. A second signal stops at once. |
|
|
79
|
+
| `SIGUSR1` | `reopen` | Reopen `log_file` (after logrotate moved it). |
|
|
80
|
+
|
|
81
|
+
`host`, `port`, `max_body_bytes`, `pid_file`, `drain_timeout` and `keep_going` need a restart.
|
|
82
|
+
A reload that changes them logs a warning and ignores them.
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
uv run vkserve -s reload -c vkserve.toml
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
A systemd unit is in [examples/vkserve.service](https://gitlab.com/cardona-oss/vkserve/-/blob/main/examples/vkserve.service).
|
|
89
|
+
|
|
90
|
+
## Features
|
|
91
|
+
|
|
92
|
+
- `POST /v1/chat/completions` and `/v1/completions`, streaming as server-sent events.
|
|
93
|
+
- Tool calls: `tools` in, OpenAI `tool_calls` out, `tool` messages back in.
|
|
94
|
+
- Several requests in one batched forward pass per step.
|
|
95
|
+
- Several GPUs: one worker per device, least-busy routing, failed workers skipped and restarted.
|
|
96
|
+
- Operations: `429` when the queue is full, `X-Request-ID`, Prometheus `/metrics` per device.
|
|
97
|
+
- Prefix reuse: a request that starts with a cached conversation prefills only the new tokens.
|
|
98
|
+
- Attention over long contexts (tested to 20,000 keys).
|
|
99
|
+
|
|
100
|
+
More: [docs/serving.md](https://gitlab.com/cardona-oss/vkserve/-/blob/main/docs/serving.md) (design and measurements).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# libvkserve_engine: the multi-sequence engine, loaded by src/vkserve/engine.py through ctypes.
|
|
2
|
+
# It builds compute-ml from source (COMPUTE_ML_DIR, default ../compute-ml next to this repo) and
|
|
3
|
+
# compiles compute-ml's LLM core (examples/llm) into the same library.
|
|
4
|
+
cmake_minimum_required(VERSION 3.21)
|
|
5
|
+
project(vkserve_engine LANGUAGES C)
|
|
6
|
+
set(CMAKE_C_STANDARD 11)
|
|
7
|
+
set(CMAKE_C_STANDARD_REQUIRED ON)
|
|
8
|
+
|
|
9
|
+
set(COMPUTE_ML_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../compute-ml" CACHE PATH "compute-ml source tree")
|
|
10
|
+
get_filename_component(COMPUTE_ML_DIR "${COMPUTE_ML_DIR}" ABSOLUTE)
|
|
11
|
+
if(NOT EXISTS "${COMPUTE_ML_DIR}/examples/llm/llm.c")
|
|
12
|
+
message(FATAL_ERROR "compute-ml not found at ${COMPUTE_ML_DIR}; pass -DCOMPUTE_ML_DIR=<path>")
|
|
13
|
+
endif()
|
|
14
|
+
add_subdirectory("${COMPUTE_ML_DIR}" compute-ml EXCLUDE_FROM_ALL)
|
|
15
|
+
find_package(Vulkan REQUIRED)
|
|
16
|
+
|
|
17
|
+
add_library(vkserve_engine SHARED engine.c devices.c
|
|
18
|
+
"${COMPUTE_ML_DIR}/examples/llm/llm.c" "${COMPUTE_ML_DIR}/examples/whisper/json.c")
|
|
19
|
+
target_include_directories(vkserve_engine PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
|
|
20
|
+
"${COMPUTE_ML_DIR}/examples/llm" "${COMPUTE_ML_DIR}/examples/whisper")
|
|
21
|
+
target_link_libraries(vkserve_engine PRIVATE compute_ml Vulkan::Vulkan)
|
|
22
|
+
if(NOT MSVC)
|
|
23
|
+
target_link_libraries(vkserve_engine PRIVATE m)
|
|
24
|
+
endif()
|
|
25
|
+
set_target_properties(vkserve_engine PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/* List the Vulkan devices that can run compute, in the order compute-ml numbers them
|
|
2
|
+
* (the index CML_GPU_INDEX takes). */
|
|
3
|
+
|
|
4
|
+
#include <stdio.h>
|
|
5
|
+
#include <stdlib.h>
|
|
6
|
+
#include <string.h>
|
|
7
|
+
|
|
8
|
+
#include <vulkan/vulkan.h>
|
|
9
|
+
|
|
10
|
+
static const char *kind_name(VkPhysicalDeviceType t) {
|
|
11
|
+
switch (t) {
|
|
12
|
+
case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
|
|
13
|
+
case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
|
|
14
|
+
case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
|
|
15
|
+
case VK_PHYSICAL_DEVICE_TYPE_CPU: return "cpu";
|
|
16
|
+
default: return "other";
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
static int has_compute(VkPhysicalDevice d) {
|
|
21
|
+
uint32_t n = 0;
|
|
22
|
+
vkGetPhysicalDeviceQueueFamilyProperties(d, &n, NULL);
|
|
23
|
+
VkQueueFamilyProperties *q = malloc((n ? n : 1) * sizeof *q);
|
|
24
|
+
if (!q) return 0;
|
|
25
|
+
vkGetPhysicalDeviceQueueFamilyProperties(d, &n, q);
|
|
26
|
+
int ok = 0;
|
|
27
|
+
for (uint32_t i = 0; i < n && !ok; i++) ok = (q[i].queueFlags & VK_QUEUE_COMPUTE_BIT) != 0;
|
|
28
|
+
free(q);
|
|
29
|
+
return ok;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/* Write one line per device into buf: "<index>\t<kind>\t<compute 0|1>\t<name>\n". Returns the
|
|
33
|
+
* number of devices, or -1 when Vulkan cannot start. buf is always NUL-terminated. */
|
|
34
|
+
int vkserve_devices(char *buf, size_t cap) {
|
|
35
|
+
if (!buf || !cap) return -1;
|
|
36
|
+
buf[0] = 0;
|
|
37
|
+
VkApplicationInfo app = { .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
|
|
38
|
+
.apiVersion = VK_API_VERSION_1_1 };
|
|
39
|
+
VkInstanceCreateInfo ci = { .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
|
|
40
|
+
.pApplicationInfo = &app };
|
|
41
|
+
VkInstance inst;
|
|
42
|
+
if (vkCreateInstance(&ci, NULL, &inst) != VK_SUCCESS) return -1;
|
|
43
|
+
uint32_t n = 0;
|
|
44
|
+
vkEnumeratePhysicalDevices(inst, &n, NULL);
|
|
45
|
+
VkPhysicalDevice *devs = malloc((n ? n : 1) * sizeof *devs);
|
|
46
|
+
if (!devs) { vkDestroyInstance(inst, NULL); return -1; }
|
|
47
|
+
vkEnumeratePhysicalDevices(inst, &n, devs);
|
|
48
|
+
size_t used = 0;
|
|
49
|
+
for (uint32_t i = 0; i < n; i++) {
|
|
50
|
+
VkPhysicalDeviceProperties p;
|
|
51
|
+
vkGetPhysicalDeviceProperties(devs[i], &p);
|
|
52
|
+
int w = snprintf(buf + used, cap - used, "%u\t%s\t%d\t%s\n", i, kind_name(p.deviceType),
|
|
53
|
+
has_compute(devs[i]), p.deviceName);
|
|
54
|
+
if (w < 0 || (size_t)w >= cap - used) break;
|
|
55
|
+
used += (size_t)w;
|
|
56
|
+
}
|
|
57
|
+
free(devs);
|
|
58
|
+
vkDestroyInstance(inst, NULL);
|
|
59
|
+
return (int)n;
|
|
60
|
+
}
|