py3model 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py3model-0.1.0/.mailmap +1 -0
- py3model-0.1.0/AUTHORS.txt +6 -0
- py3model-0.1.0/CHANGELOG +17 -0
- py3model-0.1.0/LICENSE +21 -0
- py3model-0.1.0/MANIFEST.in +7 -0
- py3model-0.1.0/PKG-INFO +70 -0
- py3model-0.1.0/README.md +38 -0
- py3model-0.1.0/docs/api.md +221 -0
- py3model-0.1.0/docs/config.md +89 -0
- py3model-0.1.0/docs/install.md +50 -0
- py3model-0.1.0/logo.jpeg +0 -0
- py3model-0.1.0/setup.cfg +4 -0
- py3model-0.1.0/setup.py +43 -0
- py3model-0.1.0/src/py3model/__init__.py +1 -0
- py3model-0.1.0/src/py3model/cli.py +89 -0
- py3model-0.1.0/src/py3model/config/__init__.py +2 -0
- py3model-0.1.0/src/py3model/config/index.py +102 -0
- py3model-0.1.0/src/py3model/gguf/__init__.py +3 -0
- py3model-0.1.0/src/py3model/gguf/index.py +429 -0
- py3model-0.1.0/src/py3model/gguf/stream.py +274 -0
- py3model-0.1.0/src/py3model/gui/__init__.py +2 -0
- py3model-0.1.0/src/py3model/gui/index.py +1206 -0
- py3model-0.1.0/src/py3model/server/__init__.py +2 -0
- py3model-0.1.0/src/py3model/server/app.py +150 -0
- py3model-0.1.0/src/py3model/server/index.py +90 -0
- py3model-0.1.0/src/py3model/server/routes/__init__.py +37 -0
- py3model-0.1.0/src/py3model/server/routes/chat.py +76 -0
- py3model-0.1.0/src/py3model/server/routes/completion.py +63 -0
- py3model-0.1.0/src/py3model/server/routes/health.py +17 -0
- py3model-0.1.0/src/py3model/server/routes/internal.py +81 -0
- py3model-0.1.0/src/py3model/server/routes/models.py +28 -0
- py3model-0.1.0/src/py3model/server/sse.py +41 -0
- py3model-0.1.0/src/py3model.egg-info/PKG-INFO +70 -0
- py3model-0.1.0/src/py3model.egg-info/SOURCES.txt +36 -0
- py3model-0.1.0/src/py3model.egg-info/dependency_links.txt +1 -0
- py3model-0.1.0/src/py3model.egg-info/entry_points.txt +2 -0
- py3model-0.1.0/src/py3model.egg-info/requires.txt +1 -0
- py3model-0.1.0/src/py3model.egg-info/top_level.txt +1 -0
py3model-0.1.0/.mailmap
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
zxl20070701 <1904314465@qq.com>
|
py3model-0.1.0/CHANGELOG
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
v0.1.0:
|
|
2
|
+
date:2026-09-02
|
|
3
|
+
changes:
|
|
4
|
+
- 初始化版本
|
|
5
|
+
1、提供 py3model 命令行工具,安装后执行 `py3model` 即可启动本地 GGUF 模型服务
|
|
6
|
+
2、内置 tkinter 可视化界面,支持选择 GGUF 文件、配置推理参数(基础 + 高级,分组折叠展示)、查看服务状态、复制 base_url、内置多轮测试对话(支持流式输出)
|
|
7
|
+
3、内置基于标准库 http.server 的 ThreadingHTTPServer,提供 OpenAI 兼容的 HTTP 接口:
|
|
8
|
+
- POST /v1/chat/completions(支持流式 SSE 与非流式)
|
|
9
|
+
- POST /v1/completions
|
|
10
|
+
- GET /v1/models
|
|
11
|
+
- GET /health
|
|
12
|
+
- POST /v1/internal/load
|
|
13
|
+
- POST /v1/internal/unload
|
|
14
|
+
4、封装 llama_cpp.Llama 为单例加载/卸载(load_model / unload_model),维护 LLAMA_PARAMS 参数元数据(含默认值、类型、悬浮提示说明)
|
|
15
|
+
5、推理层使用 ChatML 模板拼接 messages,并提供 stream_chat / non_stream_chat / stream_complete / non_stream_complete
|
|
16
|
+
6、支持 ~/.py3model/config.json 持久化配置(模型路径、推理参数、host、port 等),原子写入并向前兼容
|
|
17
|
+
7、跨平台支持(Windows / macOS / Linux),含 HiDPI 自适应与窗口最大化处理;支持 --host、--port、--no-gui 命令行参数
|
py3model-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) zxl20070701 走一步,再走一步
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
py3model-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: py3model
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: 提供python3版本的本地模型文件运行脚本或命令
|
|
5
|
+
Home-page: https://github.com/aihatch/py3model
|
|
6
|
+
Author: zxl20070701
|
|
7
|
+
Author-email: 1904314465@qq.com
|
|
8
|
+
Maintainer: zxl20070701
|
|
9
|
+
Maintainer-email: 1904314465@qq.com
|
|
10
|
+
License: MIT
|
|
11
|
+
Keywords: gguf,model,python3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
License-File: AUTHORS.txt
|
|
18
|
+
Requires-Dist: llama-cpp-python==0.3.23
|
|
19
|
+
Dynamic: author
|
|
20
|
+
Dynamic: author-email
|
|
21
|
+
Dynamic: classifier
|
|
22
|
+
Dynamic: description
|
|
23
|
+
Dynamic: description-content-type
|
|
24
|
+
Dynamic: home-page
|
|
25
|
+
Dynamic: keywords
|
|
26
|
+
Dynamic: license
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
Dynamic: maintainer
|
|
29
|
+
Dynamic: maintainer-email
|
|
30
|
+
Dynamic: requires-dist
|
|
31
|
+
Dynamic: summary
|
|
32
|
+
|
|
33
|
+
<img src="./logo.jpeg" />
|
|
34
|
+
|
|
35
|
+
# [py3model](https://github.com/aihatch/py3model)
|
|
36
|
+
⏰ 提供python3版本的本地模型文件运行脚本或命令
|
|
37
|
+
|
|
38
|
+
<p>
|
|
39
|
+
<a href="https://github.com/aihatch/py3model" target='_blank'>
|
|
40
|
+
<img alt="GitHub repo stars" src="https://img.shields.io/github/stars/aihatch/py3model?style=social">
|
|
41
|
+
</a>
|
|
42
|
+
</p>
|
|
43
|
+
|
|
44
|
+
## 使用
|
|
45
|
+
|
|
46
|
+
首先需要进行安装:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
pip3 install py3model
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
然后就可以使用了,只需要打开命令行执行:
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
py3model
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
就可以出现可视化界面,帮助你快速本地启动下载的GGUF等模型文件,作为LLM服务器使用。
|
|
59
|
+
|
|
60
|
+
## 文档
|
|
61
|
+
|
|
62
|
+
- [安装与启动](https://github.com/aihatch/py3model/blob/master/docs/install.md)
|
|
63
|
+
- [配置文件](https://github.com/aihatch/py3model/blob/master/docs/config.md)
|
|
64
|
+
- [HTTP API(OpenAI 兼容)](https://github.com/aihatch/py3model/blob/master/docs/api.md)
|
|
65
|
+
|
|
66
|
+
## 版权
|
|
67
|
+
|
|
68
|
+
MIT License
|
|
69
|
+
|
|
70
|
+
Copyright (c) [zxl20070701](https://zxl20070701.github.io/notebook/home.html) 走一步,再走一步
|
py3model-0.1.0/README.md
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
<img src="./logo.jpeg" />
|
|
2
|
+
|
|
3
|
+
# [py3model](https://github.com/aihatch/py3model)
|
|
4
|
+
⏰ 提供python3版本的本地模型文件运行脚本或命令
|
|
5
|
+
|
|
6
|
+
<p>
|
|
7
|
+
<a href="https://github.com/aihatch/py3model" target='_blank'>
|
|
8
|
+
<img alt="GitHub repo stars" src="https://img.shields.io/github/stars/aihatch/py3model?style=social">
|
|
9
|
+
</a>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
## 使用
|
|
13
|
+
|
|
14
|
+
首先需要进行安装:
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
pip3 install py3model
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
然后就可以使用了,只需要打开命令行执行:
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
py3model
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
就可以出现可视化界面,帮助你快速本地启动下载的GGUF等模型文件,作为LLM服务器使用。
|
|
27
|
+
|
|
28
|
+
## 文档
|
|
29
|
+
|
|
30
|
+
- [安装与启动](./docs/install.md)
|
|
31
|
+
- [配置文件](./docs/config.md)
|
|
32
|
+
- [HTTP API(OpenAI 兼容)](./docs/api.md)
|
|
33
|
+
|
|
34
|
+
## 版权
|
|
35
|
+
|
|
36
|
+
MIT License
|
|
37
|
+
|
|
38
|
+
Copyright (c) [zxl20070701](https://zxl20070701.github.io/notebook/home.html) 走一步,再走一步
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# HTTP API
|
|
2
|
+
|
|
3
|
+
base_url:`http://127.0.0.1:11434`
|
|
4
|
+
|
|
5
|
+
兼容 OpenAI 接口协议,可直接用 OpenAI Python SDK、LangChain、Cursor 等工具接入。
|
|
6
|
+
|
|
7
|
+
## 公共约定
|
|
8
|
+
|
|
9
|
+
- 请求/响应均为 JSON(流式接口使用 SSE)
|
|
10
|
+
- `Content-Type: application/json`
|
|
11
|
+
- 所有错误响应形如:`{"error": {"message": "...", "type": "..."}}`
|
|
12
|
+
|
|
13
|
+
### 错误码
|
|
14
|
+
|
|
15
|
+
| 状态码 | `type` | 触发场景 |
|
|
16
|
+
|---|---|---|
|
|
17
|
+
| 400 | `bad_request` | 请求体不是合法 JSON / 缺必填字段(`messages`、`prompt` 等) |
|
|
18
|
+
| 400 | `bad_request` | `/v1/internal/load` 的 `model_path` 无效或非整数参数 |
|
|
19
|
+
| 404 | `not_found` | 未匹配的路由 |
|
|
20
|
+
| 500 | `internal_error` | 推理时未预期的异常 |
|
|
21
|
+
| 500 | `load_error` | `/v1/internal/load` 加载模型失败 |
|
|
22
|
+
| 503 | `model_not_loaded` | 尚未加载模型就请求推理 |
|
|
23
|
+
|
|
24
|
+
### 模板说明
|
|
25
|
+
|
|
26
|
+
`/v1/chat/completions` 内部使用 **ChatML** 模板拼接 `messages`,并以 `` ` `` 包裹的特殊 token `<|im_end|>` 作为默认停止符(兼容 Qwen、多数中文模型):
|
|
27
|
+
|
|
28
|
+
```text
|
|
29
|
+
<|im_start|>system
|
|
30
|
+
{content}<|im_end|>
|
|
31
|
+
<|im_start|>user
|
|
32
|
+
{content}<|im_end|>
|
|
33
|
+
<|im_start|>assistant
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`/v1/completions` 则直接把 `prompt` 字段透传给 llama.cpp,不加任何额外模板。
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## POST /v1/chat/completions
|
|
41
|
+
|
|
42
|
+
聊天补全。OpenAI 兼容,支持流式与非流式。
|
|
43
|
+
|
|
44
|
+
### 请求体
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"messages": [
|
|
49
|
+
{"role": "system", "content": "你是一个有帮助的助手。"},
|
|
50
|
+
{"role": "user", "content": "你好"}
|
|
51
|
+
],
|
|
52
|
+
"temperature": 0.7,
|
|
53
|
+
"max_tokens": 1024,
|
|
54
|
+
"stream": true
|
|
55
|
+
}
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
| 字段 | 类型 | 必填 | 说明 |
|
|
59
|
+
|---|---|---|---|
|
|
60
|
+
| `messages` | array | 是 | 消息列表,每项含 `role` 与 `content` |
|
|
61
|
+
| `temperature` | float | 否 | 采样温度,默认 0.7 |
|
|
62
|
+
| `max_tokens` | int | 否 | 最大生成 token 数,默认 1024 |
|
|
63
|
+
| `stream` | bool | 否 | 是否流式输出,默认 false |
|
|
64
|
+
| `stop` | string 或 array | 否 | 停止符,默认 `<|im_end|>` |
|
|
65
|
+
|
|
66
|
+
### 非流式响应
|
|
67
|
+
|
|
68
|
+
```json
|
|
69
|
+
{
|
|
70
|
+
"id": "chatcmpl-...",
|
|
71
|
+
"object": "chat.completion",
|
|
72
|
+
"created": 1700000000,
|
|
73
|
+
"model": "gguf-local",
|
|
74
|
+
"choices": [{
|
|
75
|
+
"index": 0,
|
|
76
|
+
"message": {"role": "assistant", "content": "你好!"},
|
|
77
|
+
"finish_reason": "stop"
|
|
78
|
+
}],
|
|
79
|
+
"usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
> 说明:`usage` 字段目前全部填 0,llama-cpp-python 未直接暴露 token 计数。
|
|
84
|
+
|
|
85
|
+
### 流式响应(SSE)
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant"},"index":0,"finish_reason":null}]}
|
|
89
|
+
|
|
90
|
+
data: {"id":"...","object":"chat.completion.chunk","choices":[{"delta":{"content":"你"},"index":0,"finish_reason":null}]}
|
|
91
|
+
|
|
92
|
+
...
|
|
93
|
+
|
|
94
|
+
data: [DONE]
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## POST /v1/completions
|
|
100
|
+
|
|
101
|
+
文本补全。
|
|
102
|
+
|
|
103
|
+
### 请求体
|
|
104
|
+
|
|
105
|
+
```json
|
|
106
|
+
{
|
|
107
|
+
"prompt": "Q: 你叫什么?\nA:",
|
|
108
|
+
"temperature": 0.7,
|
|
109
|
+
"max_tokens": 512,
|
|
110
|
+
"stream": false
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
| 字段 | 类型 | 必填 | 说明 |
|
|
115
|
+
|---|---|---|---|
|
|
116
|
+
| `prompt` | string | 是 | 输入提示词 |
|
|
117
|
+
| `temperature` | float | 否 | 采样温度,默认 0.7 |
|
|
118
|
+
| `max_tokens` | int | 否 | 最大生成 token 数,默认 1024 |
|
|
119
|
+
| `stream` | bool | 否 | 是否流式输出,默认 false |
|
|
120
|
+
| `stop` | string 或 array | 否 | 停止符,**默认无**(直接透传) |
|
|
121
|
+
|
|
122
|
+
响应结构与 `/v1/chat/completions` 类似,但 `object` 为 `text_completion`,`choices` 中是 `text` 字段而非 `message`。
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## GET /v1/models
|
|
127
|
+
|
|
128
|
+
列出已加载的模型。
|
|
129
|
+
|
|
130
|
+
```json
|
|
131
|
+
{
|
|
132
|
+
"object": "list",
|
|
133
|
+
"data": [
|
|
134
|
+
{"id": "Qwen3.5-9B-Q5_K_M.gguf", "object": "model", "created": 1700000000, "owned_by": "local", "path": "/abs/path/file.gguf"}
|
|
135
|
+
]
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
未加载模型时 `data` 为空数组。
|
|
140
|
+
|
|
141
|
+
---
|
|
142
|
+
|
|
143
|
+
## GET /health
|
|
144
|
+
|
|
145
|
+
健康检查。
|
|
146
|
+
|
|
147
|
+
```json
|
|
148
|
+
{"status": "ok", "model_loaded": true, "model_path": "/abs/path/file.gguf"}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## POST /v1/internal/load
|
|
154
|
+
|
|
155
|
+
加载模型(GUI 内部使用,也可脚本调用)。
|
|
156
|
+
|
|
157
|
+
### 请求体
|
|
158
|
+
|
|
159
|
+
```json
|
|
160
|
+
{
|
|
161
|
+
"model_path": "/abs/path/Qwen3.5-9B-Q5_K_M.gguf",
|
|
162
|
+
"n_ctx": 8192,
|
|
163
|
+
"n_threads": null,
|
|
164
|
+
"n_gpu_layers": -1
|
|
165
|
+
}
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
| 字段 | 类型 | 必填 | 说明 |
|
|
169
|
+
|---|---|---|---|
|
|
170
|
+
| `model_path` | string | 是 | .gguf 文件绝对路径 |
|
|
171
|
+
| `n_ctx` | int | 否 | 默认 8192 |
|
|
172
|
+
| `n_threads` | int/null | 否 | 默认 null,由 llama.cpp 自动选择 |
|
|
173
|
+
| `n_gpu_layers` | int | 否 | 默认 -1(全部 GPU) |
|
|
174
|
+
|
|
175
|
+
成功响应:
|
|
176
|
+
|
|
177
|
+
```json
|
|
178
|
+
{"status": "loaded", "model_path": "...", "n_ctx": 8192, "n_threads": null, "n_gpu_layers": -1}
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
加载成功后,参数会自动持久化到 `~/.py3model/config.json`。
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## POST /v1/internal/unload
|
|
186
|
+
|
|
187
|
+
卸载当前模型,释放显存/内存。
|
|
188
|
+
|
|
189
|
+
```json
|
|
190
|
+
{"status": "unloaded"}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 调用示例(OpenAI Python SDK)
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from openai import OpenAI
|
|
199
|
+
|
|
200
|
+
client = OpenAI(base_url="http://127.0.0.1:11434/v1", api_key="not-needed")
|
|
201
|
+
|
|
202
|
+
resp = client.chat.completions.create(
|
|
203
|
+
model="gguf-local",
|
|
204
|
+
messages=[{"role": "user", "content": "你好"}],
|
|
205
|
+
stream=True,
|
|
206
|
+
)
|
|
207
|
+
for chunk in resp:
|
|
208
|
+
if chunk.choices[0].delta.content:
|
|
209
|
+
print(chunk.choices[0].delta.content, end="", flush=True)
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## 调用示例(curl 流式)
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
curl http://127.0.0.1:11434/v1/chat/completions \
|
|
216
|
+
-H "Content-Type: application/json" \
|
|
217
|
+
-d '{
|
|
218
|
+
"model": "gguf-local",
|
|
219
|
+
"messages": [{"role":"user","content":"你好"}],
|
|
220
|
+
"stream": true
|
|
221
|
+
}'
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# 配置文件
|
|
2
|
+
|
|
3
|
+
配置文件位置:`~/.py3model/config.json`
|
|
4
|
+
|
|
5
|
+
首次启动会自动创建。也可以手动编辑。
|
|
6
|
+
|
|
7
|
+
## 字段说明
|
|
8
|
+
|
|
9
|
+
### 服务相关
|
|
10
|
+
|
|
11
|
+
| 字段 | 类型 | 默认值 | 说明 |
|
|
12
|
+
|---|---|---|---|
|
|
13
|
+
| `model_path` | string | `""` | 当前加载的 GGUF 模型文件绝对路径 |
|
|
14
|
+
| `host` | string | `"127.0.0.1"` | HTTP 监听地址 |
|
|
15
|
+
| `port` | int | `11434` | HTTP 监听端口 |
|
|
16
|
+
| `temperature` | float | `0.7` | 默认生成温度(生成参数,非模型加载参数) |
|
|
17
|
+
| `max_tokens` | int | `1024` | 默认最大生成 token 数(生成参数,非模型加载参数) |
|
|
18
|
+
|
|
19
|
+
### llama_cpp.Llama 加载参数
|
|
20
|
+
|
|
21
|
+
以下字段对应 `llama_cpp.Llama(...)` 构造函数的同名参数,会在加载模型时原样传入。详细解释见 GUI 中每个参数旁的 ⓘ 按钮。
|
|
22
|
+
|
|
23
|
+
| 字段 | 类型 | 默认值 | 说明 |
|
|
24
|
+
|---|---|---|---|
|
|
25
|
+
| `n_ctx` | int | `8192` | 上下文窗口大小(token 数) |
|
|
26
|
+
| `n_batch` | int | `512` | prompt eval 的逻辑最大批大小 |
|
|
27
|
+
| `n_ubatch` | int | `512` | 物理 micro-batch 大小 |
|
|
28
|
+
| `n_threads` | int/null | `null` | 文本生成的 CPU 线程数;`null`/留空表示自动 |
|
|
29
|
+
| `n_threads_batch` | int/null | `null` | prompt eval 的 CPU 线程数;`null`/留空表示自动 |
|
|
30
|
+
| `n_gpu_layers` | int | `-1` | 卸载到 GPU 的层数;`-1` = 全部 |
|
|
31
|
+
| `use_mmap` | bool | `true` | 使用 mmap 加载模型 |
|
|
32
|
+
| `use_mlock` | bool | `false` | 强制系统把模型常驻内存 |
|
|
33
|
+
| `vocab_only` | bool | `false` | 仅加载词汇表 |
|
|
34
|
+
| `logits_all` | bool | `false` | 返回所有 token 的 logits |
|
|
35
|
+
| `embedding` | bool | `false` | 开启 embedding 模式 |
|
|
36
|
+
| `offload_kqv` | bool | `true` | K/Q/V 张量卸载到 GPU |
|
|
37
|
+
| `flash_attn` | bool | `false` | 启用 Flash Attention |
|
|
38
|
+
| `op_offload` | bool/null | `null` | 操作卸载策略;`null` 表示默认 |
|
|
39
|
+
| `swa_full` | bool/null | `null` | 滑动窗口注意力 KV cache 策略 |
|
|
40
|
+
| `rope_scaling_type` | int | `-1` | RoPE 缩放类型 |
|
|
41
|
+
| `rope_freq_base` | float | `0.0` | RoPE 基础频率(0 = 使用模型自带值) |
|
|
42
|
+
| `rope_freq_scale` | float | `0.0` | RoPE 频率缩放因子 |
|
|
43
|
+
| `yarn_orig_ctx` | int | `0` | YaRN 扩展的原始训练上下文 |
|
|
44
|
+
| `yarn_ext_factor` | float | `-1.0` | YaRN 插值外推因子 |
|
|
45
|
+
| `yarn_attn_factor` | float | `1.0` | YaRN attention 缩放因子 |
|
|
46
|
+
| `yarn_beta_fast` | float | `32.0` | YaRN beta fast 参数 |
|
|
47
|
+
| `yarn_beta_slow` | float | `1.0` | YaRN beta slow 参数 |
|
|
48
|
+
| `type_k` | int/null | `null` | KV cache K 张量 GGML 类型;常见值:1=f16,2=q4_0,8=q8_0 |
|
|
49
|
+
| `type_v` | int/null | `null` | KV cache V 张量 GGML 类型;常见值:1=f16,2=q4_0,8=q8_0 |
|
|
50
|
+
| `last_n_tokens_size` | int | `64` | repeat_penalty 队列大小 |
|
|
51
|
+
| `seed` | int | `4294967295` | 随机种子(4294967295 = 随机) |
|
|
52
|
+
| `split_mode` | int | `1` | 多 GPU 模型拆分模式 |
|
|
53
|
+
| `main_gpu` | int | `0` | 主 GPU 编号 |
|
|
54
|
+
| `pooling_type` | int | `-1` | embedding 池化类型 |
|
|
55
|
+
| `attention_type` | int | `-1` | 注意力类型 |
|
|
56
|
+
| `lora_base` | string/null | `null` | LoRA base 模型路径 |
|
|
57
|
+
| `lora_scale` | float | `1.0` | LoRA 权重缩放系数 |
|
|
58
|
+
| `lora_path` | string/null | `null` | LoRA 适配器路径 |
|
|
59
|
+
| `numa` | bool | `false` | NUMA 优化 |
|
|
60
|
+
| `chat_format` | string/null | `null` | 聊天模板格式 |
|
|
61
|
+
| `spm_infill` | bool | `false` | SentencePiece infill 模式 |
|
|
62
|
+
| `no_perf` | bool | `false` | 禁用性能计时信息 |
|
|
63
|
+
| `verbose` | bool | `false` | 输出 llama.cpp 内部日志 |
|
|
64
|
+
|
|
65
|
+
## 示例
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{
|
|
69
|
+
"model_path": "/Users/me/models/Qwen3.5-9B-Q5_K_M.gguf",
|
|
70
|
+
"n_ctx": 8192,
|
|
71
|
+
"n_threads": null,
|
|
72
|
+
"n_gpu_layers": -1,
|
|
73
|
+
"temperature": 0.7,
|
|
74
|
+
"max_tokens": 1024,
|
|
75
|
+
"host": "127.0.0.1",
|
|
76
|
+
"port": 11434,
|
|
77
|
+
"use_mmap": true,
|
|
78
|
+
"offload_kqv": true,
|
|
79
|
+
"flash_attn": false
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## 修改方式
|
|
84
|
+
|
|
85
|
+
- **GUI**:在窗口「基础参数/高级参数」中调整后,点「保存参数」按钮;点「加载模型」时也会自动保存
|
|
86
|
+
- **手动**:直接编辑文件后下次启动生效
|
|
87
|
+
- **代码**:`from py3model.config import update; update(port=8080, n_ctx=4096)`
|
|
88
|
+
|
|
89
|
+
> 注:完整参数定义见源码 `src/py3model/gguf/index.py` 中的 `LLAMA_PARAMS` 字典——每个条目包含 `default / kind / desc`(描述会显示在 GUI 的悬浮提示中)。
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# 安装与启动
|
|
2
|
+
|
|
3
|
+
## 安装
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip3 install py3model
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
依赖只有 `llama-cpp-python`,会在安装时自动拉取(首次安装可能需要几分钟编译)。
|
|
10
|
+
|
|
11
|
+
## 启动
|
|
12
|
+
|
|
13
|
+
最简单的启动方式:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
py3model
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
执行后:
|
|
20
|
+
1. 命令行会打印 HTTP 服务地址
|
|
21
|
+
2. 弹出 tkinter 可视化窗口
|
|
22
|
+
3. 在窗口中选择本地 `.gguf` 文件并点击「加载模型」即可
|
|
23
|
+
|
|
24
|
+
## 命令行参数
|
|
25
|
+
|
|
26
|
+
| 参数 | 说明 | 默认值 |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| `--port` | HTTP 监听端口 | 11434(或 `~/.py3model/config.json` 中的 `port`)|
|
|
29
|
+
| `--host` | HTTP 监听地址 | `127.0.0.1`(或配置中的 `host`)|
|
|
30
|
+
| `--no-gui` | 不打开 GUI,只跑 HTTP 服务(适合远程/无桌面环境)| — |
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
py3model # 默认(读 ~/.py3model/config.json)
|
|
34
|
+
py3model --port 8080 # 指定端口
|
|
35
|
+
py3model --host 0.0.0.0 # 监听所有网卡(允许局域网访问)
|
|
36
|
+
py3model --no-gui # 只跑 HTTP 服务,不打开 GUI
|
|
37
|
+
py3model --host 0.0.0.0 --port 8080 --no-gui # 远程服务器场景
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## 系统要求
|
|
41
|
+
|
|
42
|
+
- Python 3.8+
|
|
43
|
+
- macOS / Linux / Windows 均可
|
|
44
|
+
- 8GB+ 可用内存(取决于模型大小)
|
|
45
|
+
|
|
46
|
+
## 故障排查
|
|
47
|
+
|
|
48
|
+
- **端口被占用**:用 `--port` 换一个端口,或在 GUI 启动后查看顶部状态
|
|
49
|
+
- **GGUF 文件无效**:确保文件完整下载且后缀为 `.gguf`
|
|
50
|
+
- **首次加载慢**:`llama-cpp-python` 首次加载模型需要几秒到几十秒
|
py3model-0.1.0/logo.jpeg
ADDED
|
Binary file
|
py3model-0.1.0/setup.cfg
ADDED
py3model-0.1.0/setup.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
|
|
3
|
+
from setuptools import setup, find_packages
|
|
4
|
+
|
|
5
|
+
readmeFile = open("README.md", "r", encoding="utf-8")
|
|
6
|
+
long_description = readmeFile.read().replace(
|
|
7
|
+
"./docs/", "https://github.com/aihatch/py3model/blob/master/docs/"
|
|
8
|
+
)
|
|
9
|
+
readmeFile.close()
|
|
10
|
+
|
|
11
|
+
# https://setuptools.pypa.io/en/latest/setuptools.html
|
|
12
|
+
|
|
13
|
+
setup(
|
|
14
|
+
name="py3model",
|
|
15
|
+
version="0.1.0",
|
|
16
|
+
description="提供python3版本的本地模型文件运行脚本或命令",
|
|
17
|
+
long_description=long_description,
|
|
18
|
+
long_description_content_type="text/markdown",
|
|
19
|
+
author="zxl20070701",
|
|
20
|
+
author_email="1904314465@qq.com",
|
|
21
|
+
maintainer="zxl20070701",
|
|
22
|
+
maintainer_email="1904314465@qq.com",
|
|
23
|
+
package_dir={"": "src"},
|
|
24
|
+
packages=find_packages(where="src"),
|
|
25
|
+
url="https://github.com/aihatch/py3model",
|
|
26
|
+
license="MIT",
|
|
27
|
+
keywords=["gguf","model", "python3"],
|
|
28
|
+
classifiers=[
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"License :: OSI Approved :: MIT License",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
],
|
|
33
|
+
install_requires=[
|
|
34
|
+
"llama-cpp-python==0.3.23"
|
|
35
|
+
],
|
|
36
|
+
entry_points={
|
|
37
|
+
"console_scripts": [
|
|
38
|
+
# 命令名 = 模块路径:函数名,不要加括号!
|
|
39
|
+
# 开发模式安装:pip3 install -e .
|
|
40
|
+
"py3model = py3model.cli:main",
|
|
41
|
+
]
|
|
42
|
+
}
|
|
43
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
# coding=utf-8
|
|
3
|
+
|
|
4
|
+
"""
|
|
5
|
+
py3model 命令行入口。
|
|
6
|
+
|
|
7
|
+
执行 `py3model` 时:
|
|
8
|
+
1. 解析命令行参数(端口、host 等)
|
|
9
|
+
2. 启动 HTTP 服务(后台线程)
|
|
10
|
+
3. 启动 tkinter GUI(主线程阻塞)
|
|
11
|
+
4. GUI 关闭 -> 关闭后台服务 -> 退出
|
|
12
|
+
|
|
13
|
+
用法:
|
|
14
|
+
py3model # 使用默认配置
|
|
15
|
+
py3model --port 8080 # 修改端口
|
|
16
|
+
py3model --host 0.0.0.0 # 监听所有网卡
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
from .config import load as cfg_load
|
|
23
|
+
from .server import start as server_start, stop as server_stop, is_running
|
|
24
|
+
from .gui import run as gui_run
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _parse_args() -> argparse.Namespace:
|
|
28
|
+
parser = argparse.ArgumentParser(
|
|
29
|
+
prog="py3model",
|
|
30
|
+
description="本地启动 GGUF 模型文件,并提供 OpenAI 兼容的 HTTP 服务。",
|
|
31
|
+
)
|
|
32
|
+
parser.add_argument("--host", help="HTTP 监听地址(默认读取 ~/.py3model/config.json)")
|
|
33
|
+
parser.add_argument("--port", type=int, help="HTTP 监听端口(默认 11434)")
|
|
34
|
+
parser.add_argument(
|
|
35
|
+
"--no-gui",
|
|
36
|
+
action="store_true",
|
|
37
|
+
help="不启动 GUI,只在后台运行 HTTP 服务(适合无桌面/远程服务器场景)",
|
|
38
|
+
)
|
|
39
|
+
return parser.parse_args()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def main() -> int:
|
|
43
|
+
args = _parse_args()
|
|
44
|
+
cfg = cfg_load()
|
|
45
|
+
|
|
46
|
+
host = args.host or cfg.get("host", "127.0.0.1")
|
|
47
|
+
port = args.port or int(cfg.get("port", 11434))
|
|
48
|
+
|
|
49
|
+
# 启动 HTTP 服务
|
|
50
|
+
thread, server = server_start(host=host, port=port)
|
|
51
|
+
if server is None:
|
|
52
|
+
print(
|
|
53
|
+
f"[py3model] HTTP 服务启动失败:{host}:{port} 可能已被占用。\n"
|
|
54
|
+
f" 可用 --port 修改端口,或关闭占用进程后重试。",
|
|
55
|
+
file=sys.stderr,
|
|
56
|
+
)
|
|
57
|
+
if args.no_gui:
|
|
58
|
+
return 1
|
|
59
|
+
# 即便端口被占用,仍然打开 GUI,让用户看到状态、调整后再启动
|
|
60
|
+
print("[py3model] 将继续打开 GUI,你可以更换端口后重试。", file=sys.stderr)
|
|
61
|
+
|
|
62
|
+
if is_running():
|
|
63
|
+
print(f"[py3model] HTTP 服务已启动:http://{host}:{port}")
|
|
64
|
+
print(f"[py3model] OpenAI 兼容 base_url:http://{host}:{port}/v1")
|
|
65
|
+
|
|
66
|
+
# GUI(主线程)
|
|
67
|
+
if args.no_gui:
|
|
68
|
+
try:
|
|
69
|
+
# 无 GUI 模式:阻塞主线程直到 Ctrl+C
|
|
70
|
+
import time
|
|
71
|
+
print("[py3model] 无 GUI 模式,Ctrl+C 退出。")
|
|
72
|
+
while True:
|
|
73
|
+
time.sleep(1)
|
|
74
|
+
except KeyboardInterrupt:
|
|
75
|
+
print("\n[py3model] 收到中断信号,正在关闭…")
|
|
76
|
+
finally:
|
|
77
|
+
server_stop()
|
|
78
|
+
return 0
|
|
79
|
+
|
|
80
|
+
# 默认:打开 GUI(GUI 关闭时会自动调用 server.stop)
|
|
81
|
+
try:
|
|
82
|
+
gui_run()
|
|
83
|
+
except KeyboardInterrupt:
|
|
84
|
+
server_stop()
|
|
85
|
+
return 0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
if __name__ == "__main__":
|
|
89
|
+
sys.exit(main())
|