yimo-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yimo/__init__.py +26 -0
- yimo/kit/__init__.py +98 -0
- yimo/kit/ai/__init__.py +110 -0
- yimo/kit/ai/_internal/__init__.py +27 -0
- yimo/kit/ai/_internal/failover.py +535 -0
- yimo/kit/ai/_internal/pool.py +526 -0
- yimo/kit/ai/_internal/rerank_transport.py +137 -0
- yimo/kit/ai/_internal/retry_after.py +51 -0
- yimo/kit/ai/_internal/sdk_client.py +463 -0
- yimo/kit/ai/_internal/service_base.py +115 -0
- yimo/kit/ai/_internal/types.py +28 -0
- yimo/kit/ai/agent/__init__.py +127 -0
- yimo/kit/ai/agent/approval.py +66 -0
- yimo/kit/ai/agent/errors.py +128 -0
- yimo/kit/ai/agent/loop.py +643 -0
- yimo/kit/ai/agent/mcp.py +278 -0
- yimo/kit/ai/agent/registry.py +203 -0
- yimo/kit/ai/agent/session.py +150 -0
- yimo/kit/ai/agent/skill.py +370 -0
- yimo/kit/ai/agent/types.py +161 -0
- yimo/kit/ai/chat_service.py +724 -0
- yimo/kit/ai/decision/__init__.py +57 -0
- yimo/kit/ai/decision/decima.py +853 -0
- yimo/kit/ai/decision/laya.py +552 -0
- yimo/kit/ai/embedding_service.py +198 -0
- yimo/kit/ai/exceptions.py +109 -0
- yimo/kit/ai/image_service.py +84 -0
- yimo/kit/ai/messages.py +105 -0
- yimo/kit/ai/model_builder.py +121 -0
- yimo/kit/ai/model_client.py +62 -0
- yimo/kit/ai/profiles.py +577 -0
- yimo/kit/ai/rerank_service.py +136 -0
- yimo/kit/ai/runner.py +1127 -0
- yimo/kit/common_kit.py +221 -0
- yimo/kit/config/__init__.py +44 -0
- yimo/kit/config/kit_config.py +210 -0
- yimo/kit/config/log_config.py +171 -0
- yimo/kit/crawl/__init__.py +71 -0
- yimo/kit/crawl/article_io.py +325 -0
- yimo/kit/crawl/article_models.py +172 -0
- yimo/kit/crawl/deepseek_kit.py +673 -0
- yimo/kit/crawl/web_article_kit.py +638 -0
- yimo/kit/crawl/wx_article_kit.py +692 -0
- yimo/kit/db/__init__.py +101 -0
- yimo/kit/db/audit.py +625 -0
- yimo/kit/db/base_db_kit.py +1265 -0
- yimo/kit/db/crud.py +423 -0
- yimo/kit/db/engine.py +235 -0
- yimo/kit/db/exceptions.py +45 -0
- yimo/kit/db/models/__init__.py +33 -0
- yimo/kit/db/models/base.py +56 -0
- yimo/kit/db/mysql_kit.py +348 -0
- yimo/kit/db/pg_kit.py +442 -0
- yimo/kit/db/query.py +338 -0
- yimo/kit/db/session.py +85 -0
- yimo/kit/db/sqlite_kit.py +462 -0
- yimo/kit/db/transaction.py +109 -0
- yimo/kit/docs/__init__.py +34 -0
- yimo/kit/docs/anydoc_kit.py +490 -0
- yimo/kit/docs/docx_kit.py +895 -0
- yimo/kit/docs/md2doc_kit.py +1801 -0
- yimo/kit/docs/mineru_kit.py +1497 -0
- yimo/kit/docs/rc_docparser_kit.py +654 -0
- yimo/kit/file_kit.py +518 -0
- yimo/kit/license_kit.py +533 -0
- yimo/kit/redis_kit.py +392 -0
- yimo_kit-0.1.0.dist-info/METADATA +1673 -0
- yimo_kit-0.1.0.dist-info/RECORD +70 -0
- yimo_kit-0.1.0.dist-info/WHEEL +4 -0
- yimo_kit-0.1.0.dist-info/licenses/LICENSE +201 -0
yimo/kit/ai/runner.py
ADDED
|
@@ -0,0 +1,1127 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Copyright (c) 2021-2026 溢墨修心(Yimo). All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
you may not use this file except in compliance with the License.
|
|
6
|
+
You may obtain a copy of the License at
|
|
7
|
+
|
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
|
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
See the License for the specific language governing permissions and
|
|
14
|
+
limitations under the License.
|
|
15
|
+
|
|
16
|
+
Project: yimo
|
|
17
|
+
Author: 溢墨修心(Yimo)
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
# date: 2026-08-24
|
|
23
|
+
"""yimo/kit/ai 同步门面(共享后台事件循环 + 常驻链式服务单例)
|
|
24
|
+
|
|
25
|
+
``yimo.kit.ai`` 各服务(ChatService / EmbeddingService / RerankService / ImageService)
|
|
26
|
+
均为全异步,且实例持有绑定事件循环的 ``httpx.AsyncClient``;而业务层(service)
|
|
27
|
+
多数调用点运行在同步上下文(如 ``asyncio.to_thread`` 线程、被 FastAPI 路由直调的
|
|
28
|
+
同步函数)。
|
|
29
|
+
|
|
30
|
+
本模块用 Python 3.13 标准库实现统一适配:
|
|
31
|
+
|
|
32
|
+
- 一个由 ``asyncio.Runner`` 驱动的**共享后台事件循环线程**(daemon),常驻复用,
|
|
33
|
+
避免每次调用反复创建线程池与事件循环。
|
|
34
|
+
- 通过 ``asyncio.run_coroutine_threadsafe`` 把协程从**同步上下文**(含线程、
|
|
35
|
+
``asyncio.to_thread`` 线程)投递到后台 loop 并同步阻塞取结果——无需判断
|
|
36
|
+
调用线程是否有事件循环,也规避了 ``asyncio.run()`` 在已运行 loop 内被禁的坑。
|
|
37
|
+
- 按模型类型**缓存常驻 Failover 链式服务单例**:内部经 ``AIServicePool``
|
|
38
|
+
按候选链顺序调用(优先级链 + 失败冷却 + 自动切换),并复用底层
|
|
39
|
+
``httpx.AsyncClient`` 连接池。所有 I/O 都发生在同一后台 loop。
|
|
40
|
+
|
|
41
|
+
用法::
|
|
42
|
+
|
|
43
|
+
from yimo.kit.ai import AI, shutdown
|
|
44
|
+
text = AI.chat_once(prompt) # 同步
|
|
45
|
+
vectors = AI.embed_batch(chunks) # 同步
|
|
46
|
+
hits = AI.rerank(query, docs, top_n) # 同步
|
|
47
|
+
url = AI.image(prompt) # 同步
|
|
48
|
+
AI.use("chat", "glm") # 切换 chat 当前模型档案(长期生效)
|
|
49
|
+
print(AI.available("chat")) # 查看各档案状态快照
|
|
50
|
+
AI.reload_config() # 配置热重载(改密钥/档案后无需重启)
|
|
51
|
+
# 应用退出时
|
|
52
|
+
shutdown()
|
|
53
|
+
|
|
54
|
+
注意:
|
|
55
|
+
- 本门面仅面向**同步上下文**。若调用点本身是 async 函数(事件循环内),
|
|
56
|
+
应直接 ``await`` 原生 ``ChatService.create/create_stream`` 等异步接口,
|
|
57
|
+
而非经本门面阻塞事件循环线程。
|
|
58
|
+
- ``chat_stream`` 提供同步流式迭代(每次取下一帧),适合同步线程内逐 token 消费。
|
|
59
|
+
- ``AI.use`` / ``AI.available`` 为进程内内存态(不持久化),进程重启后
|
|
60
|
+
回到 JSON 声明顺序。
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
import asyncio
|
|
64
|
+
import concurrent.futures
|
|
65
|
+
import os
|
|
66
|
+
import queue
|
|
67
|
+
import threading
|
|
68
|
+
import time
|
|
69
|
+
from collections.abc import Callable, Coroutine, Iterable, Iterator
|
|
70
|
+
from dataclasses import dataclass
|
|
71
|
+
from typing import Any, Final, TypeVar, cast
|
|
72
|
+
|
|
73
|
+
from yimo.kit.ai._internal.failover import (
|
|
74
|
+
FailoverChatService,
|
|
75
|
+
FailoverEmbeddingService,
|
|
76
|
+
FailoverImageService,
|
|
77
|
+
FailoverRerankService,
|
|
78
|
+
)
|
|
79
|
+
from yimo.kit.ai._internal.pool import AIServicePool
|
|
80
|
+
from yimo.kit.ai.chat_service import StreamStats, reset_warned
|
|
81
|
+
from yimo.kit.ai.exceptions import AIConfigError, AITimeoutError
|
|
82
|
+
from yimo.kit.ai.image_service import ImageResult
|
|
83
|
+
from yimo.kit.ai.model_builder import Service
|
|
84
|
+
from yimo.kit.ai.profiles import MODEL_TYPES, ProfileStatus, get_model_registry
|
|
85
|
+
from yimo.kit.ai.rerank_service import RerankHit
|
|
86
|
+
from yimo.kit.config.kit_config import get_settings
|
|
87
|
+
from yimo.kit.config.log_config import get_default_logger
|
|
88
|
+
|
|
89
|
+
T = TypeVar("T")
|
|
90
|
+
|
|
91
|
+
logger = get_default_logger(__name__)
|
|
92
|
+
|
|
93
|
+
# 后台常驻服务类型键
|
|
94
|
+
_CHAT = "chat"
|
|
95
|
+
_EMBEDDING = "embedding"
|
|
96
|
+
_RERANK = "rerank"
|
|
97
|
+
_IMAGE = "image"
|
|
98
|
+
|
|
99
|
+
# 支持的模型类型集合(use/available 校验与错误提示用,单一数据源见 profiles)
|
|
100
|
+
_SERVICE_TYPES: Final[frozenset[str]] = frozenset(MODEL_TYPES)
|
|
101
|
+
|
|
102
|
+
# 常驻链式服务联合类型(按模型类型缓存于后台线程)
|
|
103
|
+
FailoverService = (
|
|
104
|
+
FailoverChatService
|
|
105
|
+
| FailoverEmbeddingService
|
|
106
|
+
| FailoverRerankService
|
|
107
|
+
| FailoverImageService
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# 同步门面单次调用的默认等待上限(秒),兜底防进程退出/网络挂起;
|
|
111
|
+
# 各类型实际等待上限由 _facade_timeout 按整链预算自适应。
|
|
112
|
+
_RUN_TIMEOUT = 120.0
|
|
113
|
+
|
|
114
|
+
# 门面等待上限相对模型侧 timeout 的余量(秒)
|
|
115
|
+
_RUN_TIMEOUT_MARGIN: Final[float] = 10.0
|
|
116
|
+
|
|
117
|
+
# 流式首帧单候选等待上限(秒):在 failover.stream 内对首帧 __anext__ 施加,
|
|
118
|
+
# 超时按失败处理切下一候选;对极慢推理模型可按需调大
|
|
119
|
+
_STREAM_FIRST_FRAME_TIMEOUT: Final[float] = 60.0
|
|
120
|
+
|
|
121
|
+
# 流式帧间隔上限(秒):同步队列侧兜底,防「首帧后服务端不再吐 token」挂死
|
|
122
|
+
_STREAM_FRAME_INTERVAL_TIMEOUT: Final[float] = 120.0
|
|
123
|
+
|
|
124
|
+
# 整链预算环境变量(秒):全局覆盖所有类型的门面等待上限
|
|
125
|
+
_CHAIN_TIMEOUT_ENV: Final[str] = "KIT_AI_CHAIN_TIMEOUT"
|
|
126
|
+
|
|
127
|
+
# reload 旧 service 延迟回收宽限(秒):在途调用(含流式)的最长续用窗口
|
|
128
|
+
_DELAYED_CLOSE_SECONDS: Final[float] = 60.0
|
|
129
|
+
|
|
130
|
+
# ``chat_stream`` 流结束哨兵(放入线程安全队列)
|
|
131
|
+
_SENTINEL = object()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class _BackgroundLoop:
|
|
135
|
+
"""共享后台事件循环:单线程常驻,从任意线程安全投递协程。
|
|
136
|
+
|
|
137
|
+
Python 3.13 实现:``asyncio.Runner`` 持有并驱动一个专属事件循环,
|
|
138
|
+
``run_coroutine_threadsafe`` 负责线程间投递与结果同步。
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
def __init__(self) -> None:
|
|
142
|
+
self._lock = threading.Lock()
|
|
143
|
+
self._started = threading.Event()
|
|
144
|
+
self._thread: threading.Thread | None = None
|
|
145
|
+
self._runner: asyncio.Runner | None = None
|
|
146
|
+
self._loop: asyncio.AbstractEventLoop | None = None
|
|
147
|
+
# 后台 loop 内的 keep_alive 停止信号(线程间经 call_soon_threadsafe 操作)
|
|
148
|
+
self._keep_alive_event: asyncio.Event | None = None
|
|
149
|
+
# 仅在后台线程内读写,主线程只经 shutdown() 间接操作
|
|
150
|
+
self._services: dict[str, FailoverService] = {}
|
|
151
|
+
# AI 候选池(惰性创建;业务线程 AI.use/AI.available 与 loop 线程共享)
|
|
152
|
+
self._pool: AIServicePool | None = None
|
|
153
|
+
# 预热探测任务(进程内仅投递一次,持引用防 GC)
|
|
154
|
+
self._probe_task: asyncio.Task[None] | None = None
|
|
155
|
+
# 待探测类型子集(None=全部类型;配置重载后按失效类型重投)
|
|
156
|
+
self._probe_pending: frozenset[str] | None = None
|
|
157
|
+
# reload 旧 service 延迟回收任务(持引用防 GC)
|
|
158
|
+
self._close_tasks: set[asyncio.Task[None]] = set()
|
|
159
|
+
|
|
160
|
+
# ---------- 生命周期 ----------
|
|
161
|
+
|
|
162
|
+
def _ensure_started(self) -> asyncio.AbstractEventLoop:
|
|
163
|
+
loop = self._loop
|
|
164
|
+
if loop is not None and not loop.is_closed():
|
|
165
|
+
return loop
|
|
166
|
+
with self._lock:
|
|
167
|
+
loop = self._loop
|
|
168
|
+
if loop is not None and not loop.is_closed():
|
|
169
|
+
return loop
|
|
170
|
+
self._started = threading.Event()
|
|
171
|
+
self._thread = threading.Thread(
|
|
172
|
+
target=self._worker, name="kit-ai-runner", daemon=True
|
|
173
|
+
)
|
|
174
|
+
self._thread.start()
|
|
175
|
+
# 在 lock 外等待,避免与 worker 内获取 lock 设置 _loop 相互死锁
|
|
176
|
+
self._started.wait(timeout=5)
|
|
177
|
+
loop = self._loop
|
|
178
|
+
if loop is None:
|
|
179
|
+
raise RuntimeError("yimo/kit/ai 后台事件循环启动失败")
|
|
180
|
+
return loop
|
|
181
|
+
|
|
182
|
+
def _worker(self) -> None:
|
|
183
|
+
# Runner 需在专属线程内创建(loop 绑定创建线程)
|
|
184
|
+
runner = asyncio.Runner()
|
|
185
|
+
loop = runner.get_loop()
|
|
186
|
+
with self._lock:
|
|
187
|
+
self._runner = runner
|
|
188
|
+
self._loop = loop
|
|
189
|
+
self._keep_alive_event = asyncio.Event()
|
|
190
|
+
self._started.set()
|
|
191
|
+
try:
|
|
192
|
+
# keep_alive 阻塞等待停止信号,让 runner.run 持续处理投递的协程
|
|
193
|
+
runner.run(self._keep_alive())
|
|
194
|
+
except asyncio.CancelledError:
|
|
195
|
+
logger.debug("yimo/kit/ai 后台事件循环被取消")
|
|
196
|
+
finally:
|
|
197
|
+
self._close_services()
|
|
198
|
+
with self._lock:
|
|
199
|
+
runner.close()
|
|
200
|
+
self._runner = None
|
|
201
|
+
self._loop = None
|
|
202
|
+
self._keep_alive_event = None
|
|
203
|
+
|
|
204
|
+
async def _keep_alive(self) -> None:
|
|
205
|
+
event = self._keep_alive_event
|
|
206
|
+
if event is not None:
|
|
207
|
+
await event.wait()
|
|
208
|
+
|
|
209
|
+
def _close_services(self) -> None:
|
|
210
|
+
"""在后台线程内关闭候选池中全部常驻服务(关闭底层 AsyncClient)。
|
|
211
|
+
|
|
212
|
+
Failover 链式服务本身不持连接,连接在池内各档案的 service 上,
|
|
213
|
+
统一经 ``pool.close_all()`` 释放。
|
|
214
|
+
"""
|
|
215
|
+
loop = self._loop
|
|
216
|
+
if loop is None or loop.is_closed():
|
|
217
|
+
return
|
|
218
|
+
self._services = {}
|
|
219
|
+
pool = self._pool
|
|
220
|
+
if pool is None:
|
|
221
|
+
return
|
|
222
|
+
try:
|
|
223
|
+
loop.run_until_complete(pool.close_all())
|
|
224
|
+
except Exception as exc: # noqa: BLE001 - 关闭阶段失败不影响整体退出
|
|
225
|
+
logger.warning(f"关闭 AI 候选池失败: {exc}")
|
|
226
|
+
|
|
227
|
+
def shutdown(self) -> None:
|
|
228
|
+
"""停止后台事件循环并释放常驻服务。可在任意线程调用,幂等。"""
|
|
229
|
+
with self._lock:
|
|
230
|
+
loop = self._loop
|
|
231
|
+
thread = self._thread
|
|
232
|
+
event = self._keep_alive_event
|
|
233
|
+
# 触发 keep_alive 结束,让 runner.run 干净返回,避免 pending task 泄漏
|
|
234
|
+
if loop is not None and loop.is_running() and event is not None:
|
|
235
|
+
loop.call_soon_threadsafe(event.set)
|
|
236
|
+
if thread is not None and thread is not threading.current_thread():
|
|
237
|
+
thread.join(timeout=5)
|
|
238
|
+
|
|
239
|
+
# ---------- 候选池与当前项切换 ----------
|
|
240
|
+
|
|
241
|
+
def _get_pool(self) -> AIServicePool:
|
|
242
|
+
"""取(或首次创建)候选池:锁内单例,业务线程与 loop 线程共享。"""
|
|
243
|
+
with self._lock:
|
|
244
|
+
if self._pool is None:
|
|
245
|
+
self._pool = AIServicePool()
|
|
246
|
+
return self._pool
|
|
247
|
+
|
|
248
|
+
def _schedule_probe(self, service_type: str) -> None:
|
|
249
|
+
"""首次取服务时投递候选链预热探测(fire-and-forget,不阻塞)。
|
|
250
|
+
|
|
251
|
+
待探测集合 = 重载待探类型(``_probe_pending``,None=全部)∪ 当前
|
|
252
|
+
构建类型:冷启动为全量探测(既有行为);配置重载后按失效类型子集
|
|
253
|
+
重投,且未建过的其它类型首次构建时补齐自身预热。
|
|
254
|
+
``probe_all`` 逐类型独立容错且自身不抛异常;持引用防任务被 GC。
|
|
255
|
+
"""
|
|
256
|
+
if self._probe_task is not None:
|
|
257
|
+
return
|
|
258
|
+
pending = self._probe_pending
|
|
259
|
+
types: Iterable[str] | None = (
|
|
260
|
+
_SERVICE_TYPES if pending is None else pending | {service_type}
|
|
261
|
+
)
|
|
262
|
+
pool = self._get_pool()
|
|
263
|
+
self._probe_task = asyncio.create_task(pool.probe_all(types))
|
|
264
|
+
|
|
265
|
+
def _reset_probe(self, model_types: frozenset[str] | None) -> None:
|
|
266
|
+
"""重置预热探测:下次取服务时按 model_types(None=全部)重新投递。"""
|
|
267
|
+
self._probe_task = None
|
|
268
|
+
self._probe_pending = model_types
|
|
269
|
+
|
|
270
|
+
def use(self, model_type: str, profile: str) -> None:
|
|
271
|
+
"""切换该类型长期当前档案:指定项成为固定链头并清除冷却。
|
|
272
|
+
|
|
273
|
+
同步方法,可从任意业务线程直接调用(池内部以锁保护共享状态)。
|
|
274
|
+
|
|
275
|
+
Raises:
|
|
276
|
+
AIConfigError: 类型不存在、档案不存在或不可用
|
|
277
|
+
(未配置 / 密钥无效 / 不可达 / 因约束被排除)。
|
|
278
|
+
"""
|
|
279
|
+
self._get_pool().prefer(model_type, profile)
|
|
280
|
+
|
|
281
|
+
def available(self, model_type: str) -> list[ProfileStatus]:
|
|
282
|
+
"""返回该类型全部档案状态快照(含未配置密钥者),链头标记 current。
|
|
283
|
+
|
|
284
|
+
类型未在 ``ai_models.json`` 中配置时返回空列表。
|
|
285
|
+
"""
|
|
286
|
+
return self._get_pool().available(model_type)
|
|
287
|
+
|
|
288
|
+
# ---------- 投递 ----------
|
|
289
|
+
|
|
290
|
+
def run(
|
|
291
|
+
self,
|
|
292
|
+
coro_factory: Callable[[], Coroutine[Any, Any, T]],
|
|
293
|
+
*,
|
|
294
|
+
timeout: float = _RUN_TIMEOUT,
|
|
295
|
+
) -> T:
|
|
296
|
+
"""同步执行协程工厂:投递到后台 loop 并阻塞取结果。
|
|
297
|
+
|
|
298
|
+
因使用独立后台 loop,从同步代码、线程内均可安全调用
|
|
299
|
+
(``run_coroutine_threadsafe`` 线程安全)。
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
coro_factory: 协程工厂(每次调用产出一个新协程)
|
|
303
|
+
timeout: 同步等待上限(秒);应不低于协程内部模型侧超时
|
|
304
|
+
"""
|
|
305
|
+
loop = self._ensure_started()
|
|
306
|
+
if threading.current_thread() is self._thread:
|
|
307
|
+
# 避免在后台线程内二次同步等待造成死锁
|
|
308
|
+
raise RuntimeError("yimo/kit/ai 同步门面不支持在后台线程内再次同步调用")
|
|
309
|
+
coro = coro_factory()
|
|
310
|
+
future: concurrent.futures.Future[T] = asyncio.run_coroutine_threadsafe(
|
|
311
|
+
coro, loop
|
|
312
|
+
)
|
|
313
|
+
try:
|
|
314
|
+
return future.result(timeout=timeout)
|
|
315
|
+
except concurrent.futures.TimeoutError:
|
|
316
|
+
# 后台 loop 内的协程仍在运行,取消它避免泄漏
|
|
317
|
+
future.cancel()
|
|
318
|
+
raise AITimeoutError(f"yimo/kit/ai 同步调用超过 {timeout:g}s 未返回") from None
|
|
319
|
+
|
|
320
|
+
def submit_stream(
|
|
321
|
+
self,
|
|
322
|
+
messages: list[dict[str, str]],
|
|
323
|
+
q: queue.Queue[object],
|
|
324
|
+
*,
|
|
325
|
+
max_tokens: int | None = None,
|
|
326
|
+
thinking: bool | None = None,
|
|
327
|
+
json_mode: bool | None = None,
|
|
328
|
+
response_format: dict[str, Any] | None = None,
|
|
329
|
+
stats: StreamStats,
|
|
330
|
+
) -> concurrent.futures.Future[None]:
|
|
331
|
+
"""投递流式收集任务:后台拉取链式 ``stream`` 帧写入线程安全队列。
|
|
332
|
+
|
|
333
|
+
Args:
|
|
334
|
+
thinking: 思考开关(None 不干预);
|
|
335
|
+
结果记在 ``stats.thinking_applied``
|
|
336
|
+
json_mode: 结构化输出要求(None 不干预);
|
|
337
|
+
结果记在 ``stats.json_mode_applied``
|
|
338
|
+
response_format: 协议级原样透传(与 ``json_mode`` 互斥)
|
|
339
|
+
stats: 收尾统计累加器(由同步侧持有并读取)
|
|
340
|
+
|
|
341
|
+
返回的 future 可取消,用于同步侧中断时停止后台拉取。
|
|
342
|
+
"""
|
|
343
|
+
loop = self._ensure_started()
|
|
344
|
+
|
|
345
|
+
async def _producer() -> None:
|
|
346
|
+
service: FailoverChatService = cast(
|
|
347
|
+
FailoverChatService, await self._get_service(_CHAT)
|
|
348
|
+
)
|
|
349
|
+
try:
|
|
350
|
+
async for delta in service.create_stream(
|
|
351
|
+
messages,
|
|
352
|
+
max_tokens=max_tokens,
|
|
353
|
+
thinking=thinking,
|
|
354
|
+
json_mode=json_mode,
|
|
355
|
+
response_format=response_format,
|
|
356
|
+
first_frame_timeout=_STREAM_FIRST_FRAME_TIMEOUT,
|
|
357
|
+
stats=stats,
|
|
358
|
+
):
|
|
359
|
+
q.put(delta)
|
|
360
|
+
except BaseException as exc: # noqa: BLE001 - 异常经队列回传给同步侧
|
|
361
|
+
q.put(exc)
|
|
362
|
+
finally:
|
|
363
|
+
q.put(_SENTINEL)
|
|
364
|
+
|
|
365
|
+
return asyncio.run_coroutine_threadsafe(_producer(), loop)
|
|
366
|
+
|
|
367
|
+
async def _get_service(self, service_type: str) -> FailoverService:
|
|
368
|
+
"""后台线程内取常驻链式服务,未创建则就地构建并缓存。
|
|
369
|
+
|
|
370
|
+
进程内首次构建任意类型时投递一次候选链预热探测(fire-and-forget)。
|
|
371
|
+
"""
|
|
372
|
+
service = self._services.get(service_type)
|
|
373
|
+
if service is None:
|
|
374
|
+
self._schedule_probe(service_type)
|
|
375
|
+
pool = self._get_pool()
|
|
376
|
+
match service_type:
|
|
377
|
+
case "chat":
|
|
378
|
+
service = FailoverChatService(pool)
|
|
379
|
+
case "embedding":
|
|
380
|
+
service = FailoverEmbeddingService(pool)
|
|
381
|
+
case "rerank":
|
|
382
|
+
service = FailoverRerankService(pool)
|
|
383
|
+
case "image":
|
|
384
|
+
service = FailoverImageService(pool)
|
|
385
|
+
case _:
|
|
386
|
+
raise ValueError(
|
|
387
|
+
f"未知模型类型: {service_type},"
|
|
388
|
+
f"支持: {', '.join(sorted(_SERVICE_TYPES))}"
|
|
389
|
+
)
|
|
390
|
+
self._services[service_type] = service
|
|
391
|
+
logger.debug(f"已构建并常驻 AI 链式服务:{service_type}")
|
|
392
|
+
return service
|
|
393
|
+
|
|
394
|
+
# ---------- 配置热重载 ----------
|
|
395
|
+
|
|
396
|
+
def invalidate(
|
|
397
|
+
self, model_types: frozenset[str], *, probe: bool, immediate: bool
|
|
398
|
+
) -> None:
|
|
399
|
+
"""失效候选链与常驻服务(``reload_config`` 的执行体,同步阻塞)。
|
|
400
|
+
|
|
401
|
+
内部投递到后台 loop 执行(``_services`` / ``_probe_task`` 仅允许在
|
|
402
|
+
loop 线程内读写),返回即已生效——后续新调用必定使用新配置。
|
|
403
|
+
"""
|
|
404
|
+
self.run(
|
|
405
|
+
lambda: self._invalidate_in_loop(
|
|
406
|
+
model_types, probe=probe, immediate=immediate
|
|
407
|
+
)
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
async def _invalidate_in_loop(
|
|
411
|
+
self, model_types: frozenset[str], *, probe: bool, immediate: bool
|
|
412
|
+
) -> None:
|
|
413
|
+
"""在后台 loop 内摘链、清常驻服务并调度旧 service 关闭。
|
|
414
|
+
|
|
415
|
+
关闭语义:``immediate=True`` 逐个立即 ``close()``(在途调用按关闭
|
|
416
|
+
语义报错);默认投递延迟回收任务,给在途调用留
|
|
417
|
+
``_DELAYED_CLOSE_SECONDS`` 宽限期(service 持强引用防 GC,
|
|
418
|
+
``close`` 幂等,重复关闭无害)。
|
|
419
|
+
"""
|
|
420
|
+
pool = self._get_pool()
|
|
421
|
+
old_services = pool.invalidate(model_types)
|
|
422
|
+
for service_type in model_types:
|
|
423
|
+
self._services.pop(service_type, None)
|
|
424
|
+
if probe:
|
|
425
|
+
self._reset_probe(None if model_types == _SERVICE_TYPES else model_types)
|
|
426
|
+
if immediate:
|
|
427
|
+
for service in old_services:
|
|
428
|
+
try:
|
|
429
|
+
await service.close()
|
|
430
|
+
except Exception as exc: # noqa: BLE001 - 关闭失败不影响重载
|
|
431
|
+
logger.warning(f"关闭旧 AI service 失败: {exc}")
|
|
432
|
+
elif old_services:
|
|
433
|
+
task = asyncio.create_task(_delayed_close(old_services))
|
|
434
|
+
self._close_tasks.add(task)
|
|
435
|
+
task.add_done_callback(self._close_tasks.discard)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
_background = _BackgroundLoop()
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def run_sync[T](
|
|
442
|
+
coro_factory: Callable[[], Coroutine[Any, Any, T]],
|
|
443
|
+
*,
|
|
444
|
+
timeout: float = _RUN_TIMEOUT,
|
|
445
|
+
) -> T:
|
|
446
|
+
"""在任意上下文同步执行协程工厂(兼容旧接口,等价于后台投递)。"""
|
|
447
|
+
return _background.run(coro_factory, timeout=timeout)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _facade_timeout(model_type: str) -> float:
|
|
451
|
+
"""门面同步等待上限(秒)= 整链预算,按以下优先级取值:
|
|
452
|
+
|
|
453
|
+
1. 环境变量 ``KIT_AI_CHAIN_TIMEOUT``(全局覆盖;注册表加载后 ``.env``
|
|
454
|
+
已注入 ``os.environ``,故 .env 配置同样生效)
|
|
455
|
+
2. ``ai_models.json`` 该类型的 ``chain_timeout``
|
|
456
|
+
3. 自适应:sum(候选链各档案 timeout) + 余量——容纳「首选失败 +
|
|
457
|
+
备胎跑完」的一次完整降级
|
|
458
|
+
|
|
459
|
+
类型未配置或候选链为空时回退默认 ``_RUN_TIMEOUT``。
|
|
460
|
+
交互式业务嫌等待过长时,应配置 ``chain_timeout`` 显式收紧。
|
|
461
|
+
"""
|
|
462
|
+
try:
|
|
463
|
+
registry = get_model_registry()
|
|
464
|
+
profiles = registry.chain(model_type)
|
|
465
|
+
except AIConfigError:
|
|
466
|
+
return _RUN_TIMEOUT
|
|
467
|
+
env_value = os.environ.get(_CHAIN_TIMEOUT_ENV)
|
|
468
|
+
if env_value:
|
|
469
|
+
try:
|
|
470
|
+
return float(env_value)
|
|
471
|
+
except ValueError:
|
|
472
|
+
logger.warning(
|
|
473
|
+
f"{_CHAIN_TIMEOUT_ENV} 非法({env_value!r}),忽略,回退档案配置"
|
|
474
|
+
)
|
|
475
|
+
try:
|
|
476
|
+
configured = registry.chain_timeout(model_type)
|
|
477
|
+
except AIConfigError:
|
|
478
|
+
return _RUN_TIMEOUT
|
|
479
|
+
if configured is not None:
|
|
480
|
+
return configured
|
|
481
|
+
if not profiles:
|
|
482
|
+
return _RUN_TIMEOUT
|
|
483
|
+
return sum(float(profile.timeout) for profile in profiles) + _RUN_TIMEOUT_MARGIN
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
async def _chat(
|
|
487
|
+
messages: list[dict[str, str]],
|
|
488
|
+
*,
|
|
489
|
+
max_tokens: int | None = None,
|
|
490
|
+
thinking: bool | None = None,
|
|
491
|
+
json_mode: bool | None = None,
|
|
492
|
+
response_format: dict[str, Any] | None = None,
|
|
493
|
+
stats: StreamStats | None = None,
|
|
494
|
+
deadline: float | None = None,
|
|
495
|
+
) -> str:
|
|
496
|
+
service: FailoverChatService = cast(
|
|
497
|
+
FailoverChatService, await _background._get_service(_CHAT) # noqa: SLF001
|
|
498
|
+
)
|
|
499
|
+
return await service.create(
|
|
500
|
+
messages,
|
|
501
|
+
max_tokens=max_tokens,
|
|
502
|
+
thinking=thinking,
|
|
503
|
+
json_mode=json_mode,
|
|
504
|
+
response_format=response_format,
|
|
505
|
+
stats=stats,
|
|
506
|
+
deadline=deadline,
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
async def _chat_once(
|
|
511
|
+
prompt: str,
|
|
512
|
+
*,
|
|
513
|
+
max_tokens: int | None = None,
|
|
514
|
+
thinking: bool | None = None,
|
|
515
|
+
json_mode: bool | None = None,
|
|
516
|
+
response_format: dict[str, Any] | None = None,
|
|
517
|
+
stats: StreamStats | None = None,
|
|
518
|
+
deadline: float | None = None,
|
|
519
|
+
) -> str:
|
|
520
|
+
service: FailoverChatService = cast(
|
|
521
|
+
FailoverChatService, await _background._get_service(_CHAT) # noqa: SLF001
|
|
522
|
+
)
|
|
523
|
+
return await service.create_once(
|
|
524
|
+
prompt,
|
|
525
|
+
max_tokens=max_tokens,
|
|
526
|
+
thinking=thinking,
|
|
527
|
+
json_mode=json_mode,
|
|
528
|
+
response_format=response_format,
|
|
529
|
+
stats=stats,
|
|
530
|
+
deadline=deadline,
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
async def _embed_batch(
|
|
535
|
+
texts: list[str], *, deadline: float | None = None
|
|
536
|
+
) -> list[list[float]]:
|
|
537
|
+
if not texts:
|
|
538
|
+
return []
|
|
539
|
+
service: FailoverEmbeddingService = cast(
|
|
540
|
+
FailoverEmbeddingService,
|
|
541
|
+
await _background._get_service(_EMBEDDING), # noqa: SLF001
|
|
542
|
+
)
|
|
543
|
+
return await service.create_batch(texts, deadline=deadline)
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
async def _rerank(
|
|
547
|
+
query: str,
|
|
548
|
+
documents: list[str],
|
|
549
|
+
top_n: int | None,
|
|
550
|
+
*,
|
|
551
|
+
deadline: float | None = None,
|
|
552
|
+
) -> list[RerankHit]:
|
|
553
|
+
"""对候选文档重排序,返回 ``RerankHit`` 降序列表。"""
|
|
554
|
+
if not documents:
|
|
555
|
+
return []
|
|
556
|
+
service: FailoverRerankService = cast(
|
|
557
|
+
FailoverRerankService,
|
|
558
|
+
await _background._get_service(_RERANK), # noqa: SLF001
|
|
559
|
+
)
|
|
560
|
+
return await service.create(query, documents, top_n=top_n, deadline=deadline)
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
async def _image(
|
|
564
|
+
prompt: str,
|
|
565
|
+
*,
|
|
566
|
+
size: str | None,
|
|
567
|
+
n: int,
|
|
568
|
+
return_base64: bool = False,
|
|
569
|
+
ratio: str | None = None,
|
|
570
|
+
deadline: float | None = None,
|
|
571
|
+
) -> ImageResult:
|
|
572
|
+
service: FailoverImageService = cast(
|
|
573
|
+
FailoverImageService,
|
|
574
|
+
await _background._get_service(_IMAGE), # noqa: SLF001
|
|
575
|
+
)
|
|
576
|
+
return await service.create(
|
|
577
|
+
prompt,
|
|
578
|
+
size=size,
|
|
579
|
+
n=n,
|
|
580
|
+
return_base64=return_base64,
|
|
581
|
+
ratio=ratio,
|
|
582
|
+
deadline=deadline,
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
class AI:
|
|
587
|
+
"""yimo/kit/ai 同步门面:在同步上下文调用异步 AI 服务(候选链故障切换)。"""
|
|
588
|
+
|
|
589
|
+
@staticmethod
|
|
590
|
+
def chat_once(
|
|
591
|
+
prompt: str,
|
|
592
|
+
*,
|
|
593
|
+
max_tokens: int | None = None,
|
|
594
|
+
thinking: bool | None = None,
|
|
595
|
+
json_mode: bool | None = None,
|
|
596
|
+
response_format: dict[str, Any] | None = None,
|
|
597
|
+
stats: StreamStats | None = None,
|
|
598
|
+
) -> str:
|
|
599
|
+
"""单轮对话(仅 user 消息),返回模型回答文本。
|
|
600
|
+
|
|
601
|
+
Args:
|
|
602
|
+
max_tokens: 输出上限;None 时回落档案 ``max_tokens`` 配置,
|
|
603
|
+
仍未配置则由 provider 默认值决定
|
|
604
|
+
thinking: 思考开关;None 不干预(按档案 ``thinking.default``,
|
|
605
|
+
未配置则请求体不变)/ True 开 / False 关。档案未声明该能力时
|
|
606
|
+
忽略并告警(不报错、不影响故障切换)
|
|
607
|
+
json_mode: 结构化输出;None/False 不干预 / True 要求 JSON 对象
|
|
608
|
+
(取值按档案 ``structured_output``,未声明用内置协议默认;
|
|
609
|
+
档案写 ``false`` 时忽略并告警)
|
|
610
|
+
response_format: 协议级原样透传(含 ``json_schema`` 严格模式);
|
|
611
|
+
与 ``json_mode`` 互斥,schema 由调用方负责
|
|
612
|
+
stats: 收尾统计累加器(可选);传入后可读 ``finish_reason`` /
|
|
613
|
+
``truncated`` / ``reasoning_chars`` / ``json_mode_applied``
|
|
614
|
+
"""
|
|
615
|
+
timeout = _facade_timeout(_CHAT)
|
|
616
|
+
deadline = time.monotonic() + timeout
|
|
617
|
+
return _background.run(
|
|
618
|
+
lambda: _chat_once(
|
|
619
|
+
prompt,
|
|
620
|
+
max_tokens=max_tokens,
|
|
621
|
+
thinking=thinking,
|
|
622
|
+
json_mode=json_mode,
|
|
623
|
+
response_format=response_format,
|
|
624
|
+
stats=stats,
|
|
625
|
+
deadline=deadline,
|
|
626
|
+
),
|
|
627
|
+
timeout=timeout,
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
@staticmethod
|
|
631
|
+
def chat(
|
|
632
|
+
messages: list[dict[str, str]],
|
|
633
|
+
*,
|
|
634
|
+
max_tokens: int | None = None,
|
|
635
|
+
thinking: bool | None = None,
|
|
636
|
+
json_mode: bool | None = None,
|
|
637
|
+
response_format: dict[str, Any] | None = None,
|
|
638
|
+
stats: StreamStats | None = None,
|
|
639
|
+
) -> str:
|
|
640
|
+
"""多轮对话(自组装 messages),返回模型回答文本。
|
|
641
|
+
|
|
642
|
+
Args:
|
|
643
|
+
max_tokens: 输出上限;None 时回落档案 ``max_tokens`` 配置,
|
|
644
|
+
仍未配置则由 provider 默认值决定
|
|
645
|
+
thinking: 思考开关;语义同 :meth:`chat_once`
|
|
646
|
+
json_mode: 结构化输出要求;语义同 :meth:`chat_once`
|
|
647
|
+
response_format: 协议级原样透传;语义同 :meth:`chat_once`
|
|
648
|
+
stats: 收尾统计累加器;语义同 :meth:`chat_once`
|
|
649
|
+
"""
|
|
650
|
+
timeout = _facade_timeout(_CHAT)
|
|
651
|
+
deadline = time.monotonic() + timeout
|
|
652
|
+
return _background.run(
|
|
653
|
+
lambda: _chat(
|
|
654
|
+
messages,
|
|
655
|
+
max_tokens=max_tokens,
|
|
656
|
+
thinking=thinking,
|
|
657
|
+
json_mode=json_mode,
|
|
658
|
+
response_format=response_format,
|
|
659
|
+
stats=stats,
|
|
660
|
+
deadline=deadline,
|
|
661
|
+
),
|
|
662
|
+
timeout=timeout,
|
|
663
|
+
)
|
|
664
|
+
|
|
665
|
+
@staticmethod
|
|
666
|
+
def embed(text: str) -> list[float]:
|
|
667
|
+
"""对单个文本向量化,返回向量。"""
|
|
668
|
+
timeout = _facade_timeout(_EMBEDDING)
|
|
669
|
+
deadline = time.monotonic() + timeout
|
|
670
|
+
return _background.run(
|
|
671
|
+
lambda: _embed_batch([text], deadline=deadline), timeout=timeout
|
|
672
|
+
)[0]
|
|
673
|
+
|
|
674
|
+
@staticmethod
|
|
675
|
+
def embed_batch(texts: list[str]) -> list[list[float]]:
|
|
676
|
+
"""对文本列表批量向量化,返回向量列表。"""
|
|
677
|
+
timeout = _facade_timeout(_EMBEDDING)
|
|
678
|
+
deadline = time.monotonic() + timeout
|
|
679
|
+
return _background.run(
|
|
680
|
+
lambda: _embed_batch(texts, deadline=deadline), timeout=timeout
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
@staticmethod
|
|
684
|
+
def rerank(
|
|
685
|
+
query: str, documents: list[str], top_n: int | None = None
|
|
686
|
+
) -> list[RerankHit]:
|
|
687
|
+
"""对候选文档重排序,返回 ``RerankHit`` 降序列表(原索引、分数、原文)。
|
|
688
|
+
|
|
689
|
+
``top_n`` 是返回条数上限,由 kit 本地保证(服务端返回超量也按分数
|
|
690
|
+
截断);``None`` 返回服务端全部命中。
|
|
691
|
+
"""
|
|
692
|
+
timeout = _facade_timeout(_RERANK)
|
|
693
|
+
deadline = time.monotonic() + timeout
|
|
694
|
+
return _background.run(
|
|
695
|
+
lambda: _rerank(query, documents, top_n, deadline=deadline),
|
|
696
|
+
timeout=timeout,
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
@staticmethod
|
|
700
|
+
def image(
|
|
701
|
+
prompt: str,
|
|
702
|
+
*,
|
|
703
|
+
size: str | None = None,
|
|
704
|
+
n: int = 1,
|
|
705
|
+
return_base64: bool = False,
|
|
706
|
+
ratio: str | None = None,
|
|
707
|
+
) -> ImageResult:
|
|
708
|
+
"""文生图,返回 :class:`ImageResult`(``url`` / ``b64`` 至少一个非空)。"""
|
|
709
|
+
timeout = _facade_timeout(_IMAGE)
|
|
710
|
+
deadline = time.monotonic() + timeout
|
|
711
|
+
return _background.run(
|
|
712
|
+
lambda: _image(
|
|
713
|
+
prompt,
|
|
714
|
+
size=size,
|
|
715
|
+
n=n,
|
|
716
|
+
return_base64=return_base64,
|
|
717
|
+
ratio=ratio,
|
|
718
|
+
deadline=deadline,
|
|
719
|
+
),
|
|
720
|
+
timeout=timeout,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
@staticmethod
|
|
724
|
+
def use(model_type: str, profile: str) -> None:
|
|
725
|
+
"""切换某类型长期当前模型档案(进程内生效,重启回 env/json 序)。
|
|
726
|
+
|
|
727
|
+
指定档案成为固定链头并清除其冷却;再次 ``use`` 才会更换。
|
|
728
|
+
|
|
729
|
+
切换语义(两类,勿混淆):
|
|
730
|
+
- **跨调用切换**:``use`` 与其它请求产生的失败冷却均只影响
|
|
731
|
+
**下一次**请求——每次调用的候选计划在开始时定格;
|
|
732
|
+
- **一次调用内降级**:本次调用按开始时定格的计划串行尝试,
|
|
733
|
+
中途的 ``use`` / 冷却变化不影响它。
|
|
734
|
+
|
|
735
|
+
Args:
|
|
736
|
+
model_type: 模型类型(chat / embedding / rerank / image)
|
|
737
|
+
profile: 目标档案名(``ai_models.json`` 中该类型 profiles 的键)
|
|
738
|
+
|
|
739
|
+
Raises:
|
|
740
|
+
AIConfigError: 类型不存在、档案不存在或不可用
|
|
741
|
+
(未配置 / 密钥无效 / 不可达 / 因约束被排除)。
|
|
742
|
+
"""
|
|
743
|
+
_background.use(model_type, profile)
|
|
744
|
+
|
|
745
|
+
@staticmethod
|
|
746
|
+
def available(model_type: str) -> list[ProfileStatus]:
|
|
747
|
+
"""返回某类型全部档案状态快照(含未配置密钥与被剔除者),链头标 current。
|
|
748
|
+
|
|
749
|
+
Args:
|
|
750
|
+
model_type: 模型类型(chat / embedding / rerank / image)
|
|
751
|
+
|
|
752
|
+
Returns:
|
|
753
|
+
``ProfileStatus`` 列表(name/model/state/current/last_error/
|
|
754
|
+
last_ok_at),供业务展示与切换决策。**类型在 ``ai_models.json``
|
|
755
|
+
中未配置时返回空列表**(可预期空态,供界面显示「未配置」)。
|
|
756
|
+
|
|
757
|
+
Raises:
|
|
758
|
+
AIConfigError: 类型名非法(不在四类之内)或注册表未加载。
|
|
759
|
+
"""
|
|
760
|
+
return _background.available(model_type)
|
|
761
|
+
|
|
762
|
+
@staticmethod
|
|
763
|
+
def reload_config(
|
|
764
|
+
*,
|
|
765
|
+
model_type: str | None = None,
|
|
766
|
+
reload_file: bool = False,
|
|
767
|
+
probe: bool = True,
|
|
768
|
+
immediate: bool = False,
|
|
769
|
+
) -> ReloadResult:
|
|
770
|
+
"""配置热重载:失效候选链与常驻服务,新配置对后续调用立即生效。
|
|
771
|
+
|
|
772
|
+
参数与并发契约详见模块级 :func:`reload_config`。
|
|
773
|
+
"""
|
|
774
|
+
return reload_config(
|
|
775
|
+
model_type=model_type,
|
|
776
|
+
reload_file=reload_file,
|
|
777
|
+
probe=probe,
|
|
778
|
+
immediate=immediate,
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
@staticmethod
|
|
782
|
+
def chat_stream(
|
|
783
|
+
messages: list[dict[str, str]],
|
|
784
|
+
*,
|
|
785
|
+
max_tokens: int | None = None,
|
|
786
|
+
thinking: bool | None = None,
|
|
787
|
+
json_mode: bool | None = None,
|
|
788
|
+
response_format: dict[str, Any] | None = None,
|
|
789
|
+
) -> Iterator[str]:
|
|
790
|
+
"""多轮对话(流式),返回逐 token 迭代器(同步)。
|
|
791
|
+
|
|
792
|
+
在同步线程内逐帧消费,适合流式输出;内部在后台 loop 持续拉取
|
|
793
|
+
并放入线程安全队列。
|
|
794
|
+
|
|
795
|
+
Args:
|
|
796
|
+
thinking: 思考开关;None 不干预(按档案 ``thinking.default``)/
|
|
797
|
+
True 开 / False 关。是否真正生效读 ``stream.thinking_applied``
|
|
798
|
+
(None 未干预 / True 已注入 / False 被忽略)
|
|
799
|
+
json_mode: 结构化输出要求;None 不干预 / True 要求 JSON 对象。
|
|
800
|
+
是否真正注入读 ``stream.json_mode_applied``;流式下 JSON 只能是
|
|
801
|
+
半成品,kit 不做增量解析
|
|
802
|
+
response_format: 协议级原样透传(与 ``json_mode`` 互斥)
|
|
803
|
+
|
|
804
|
+
超时保护:
|
|
805
|
+
- 首帧前:每候选 ``_STREAM_FIRST_FRAME_TIMEOUT``(60s)内**一帧
|
|
806
|
+
都没有**(含推理帧)则切下一候选,全部候选耗尽抛
|
|
807
|
+
``AITimeoutError``;思考型模型的推理帧算活动、会重置窗口;
|
|
808
|
+
- 首帧后:帧间隔超过 ``_STREAM_FRAME_INTERVAL_TIMEOUT``(120s)
|
|
809
|
+
未收到新帧则中断并抛 ``AITimeoutError``(已产出 token 不可回滚)。
|
|
810
|
+
|
|
811
|
+
推荐配合 ``with`` 使用,保证中途 ``break`` 也会及时取消后台任务::
|
|
812
|
+
|
|
813
|
+
with AI.chat_stream(messages) as stream:
|
|
814
|
+
for token in stream:
|
|
815
|
+
yield token
|
|
816
|
+
if done:
|
|
817
|
+
break
|
|
818
|
+
|
|
819
|
+
流结束后可读八个只读属性:``reasoning_chars`` / ``content_chars`` /
|
|
820
|
+
``finish_reason`` / ``requested_max_tokens`` / ``truncated`` / ``empty_kind`` /
|
|
821
|
+
``thinking_applied`` / ``json_mode_applied``,用于区分「模型没返回内容」与
|
|
822
|
+
「输出预算被思考吃光」(``empty_kind`` 给出五类归因),以及确认开关是否真的生效::
|
|
823
|
+
|
|
824
|
+
with AI.chat_stream(messages, max_tokens=450) as stream:
|
|
825
|
+
body = "".join(stream)
|
|
826
|
+
if not body and stream.truncated and stream.reasoning_chars:
|
|
827
|
+
... # 推理吃光预算:调大 max_tokens 或传 thinking=False
|
|
828
|
+
"""
|
|
829
|
+
return _StreamIterator(
|
|
830
|
+
messages,
|
|
831
|
+
max_tokens=max_tokens,
|
|
832
|
+
thinking=thinking,
|
|
833
|
+
json_mode=json_mode,
|
|
834
|
+
response_format=response_format,
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
class _StreamIterator:
|
|
839
|
+
"""``AI.chat_stream`` 的同步流式迭代器。
|
|
840
|
+
|
|
841
|
+
通过 ``queue.Queue`` 桥接后台 loop:一个后台任务把链式 ``stream``
|
|
842
|
+
产出的帧逐条放入队列,同步侧阻塞 ``get`` 消费;停止信号由哨兵标记。
|
|
843
|
+
首帧等待由后台逐候选首帧预算保证有界;首帧之后的 ``get`` 带
|
|
844
|
+
帧间隔超时,防服务端停摆时永久阻塞。
|
|
845
|
+
|
|
846
|
+
支持 ``with`` 上下文(推荐):退出时主动取消后台任务;即便调用方
|
|
847
|
+
直接 ``for`` 中途 ``break`` 或迭代器被 GC,``close``/``__del__`` 也会
|
|
848
|
+
兜底取消后台 producer,避免任务残留。
|
|
849
|
+
|
|
850
|
+
流结束后可读八个只读属性:``reasoning_chars`` / ``content_chars`` /
|
|
851
|
+
``finish_reason`` / ``requested_max_tokens`` / ``truncated`` / ``empty_kind`` /
|
|
852
|
+
``thinking_applied`` / ``json_mode_applied``(跨线程共享同一累加器:
|
|
853
|
+
结束后为最终值,流式过程中为尽力而为的中间值;提前 ``break`` 时为截至
|
|
854
|
+
中断的累计值)。
|
|
855
|
+
|
|
856
|
+
``empty_kind`` 例外:**仅在流正常结束后可读**,迭代中与异常 / 中断后
|
|
857
|
+
恒为 ``None``(避免把「正文还没到」误判成 ``no_finish``)。
|
|
858
|
+
"""
|
|
859
|
+
|
|
860
|
+
def __init__(
|
|
861
|
+
self,
|
|
862
|
+
messages: list[dict[str, str]],
|
|
863
|
+
*,
|
|
864
|
+
max_tokens: int | None = None,
|
|
865
|
+
thinking: bool | None = None,
|
|
866
|
+
json_mode: bool | None = None,
|
|
867
|
+
response_format: dict[str, Any] | None = None,
|
|
868
|
+
) -> None:
|
|
869
|
+
self._messages: list[dict[str, str]] = messages
|
|
870
|
+
self._max_tokens: int | None = max_tokens
|
|
871
|
+
self._thinking: bool | None = thinking
|
|
872
|
+
self._json_mode: bool | None = json_mode
|
|
873
|
+
self._response_format: dict[str, Any] | None = response_format
|
|
874
|
+
self._q: queue.Queue[object] = queue.Queue()
|
|
875
|
+
self._stats: StreamStats = StreamStats()
|
|
876
|
+
self._started = False
|
|
877
|
+
self._closed = False
|
|
878
|
+
self._got_first = False
|
|
879
|
+
# 正常收尾(收到哨兵)才置真:empty_kind 只在结束后给结论
|
|
880
|
+
self._exhausted = False
|
|
881
|
+
self._task: concurrent.futures.Future[None] | None = None
|
|
882
|
+
|
|
883
|
+
def __iter__(self) -> Iterator[str]:
|
|
884
|
+
return self
|
|
885
|
+
|
|
886
|
+
def _ensure_started(self) -> None:
|
|
887
|
+
if not self._started:
|
|
888
|
+
self._started = True
|
|
889
|
+
self._task = _background.submit_stream(
|
|
890
|
+
self._messages,
|
|
891
|
+
self._q,
|
|
892
|
+
max_tokens=self._max_tokens,
|
|
893
|
+
thinking=self._thinking,
|
|
894
|
+
json_mode=self._json_mode,
|
|
895
|
+
response_format=self._response_format,
|
|
896
|
+
stats=self._stats,
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
def _cancel_task(self) -> None:
|
|
900
|
+
task = self._task
|
|
901
|
+
if task is not None:
|
|
902
|
+
task.cancel()
|
|
903
|
+
self._task = None
|
|
904
|
+
|
|
905
|
+
def close(self) -> None:
|
|
906
|
+
"""停止后台拉取并释放资源。幂等,可多次调用。"""
|
|
907
|
+
if self._closed:
|
|
908
|
+
return
|
|
909
|
+
self._closed = True
|
|
910
|
+
self._cancel_task()
|
|
911
|
+
|
|
912
|
+
def __del__(self) -> None:
|
|
913
|
+
# 兜底清理:for 中途 break / 迭代器被 GC 时取消后台 producer。
|
|
914
|
+
# 模块卸载期 _task 是独立 future 引用,cancel 对已 shutdown 的 loop 安全。
|
|
915
|
+
try:
|
|
916
|
+
self.close()
|
|
917
|
+
except Exception: # noqa: BLE001 - GC 阶段不容许抛异常
|
|
918
|
+
pass
|
|
919
|
+
|
|
920
|
+
def __enter__(self) -> _StreamIterator:
|
|
921
|
+
return self
|
|
922
|
+
|
|
923
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
924
|
+
self.close()
|
|
925
|
+
|
|
926
|
+
def __next__(self) -> str:
|
|
927
|
+
if self._closed:
|
|
928
|
+
raise StopIteration
|
|
929
|
+
self._ensure_started()
|
|
930
|
+
# 首帧等待由后台逐候选首帧预算保证有界,不设队列级超时;
|
|
931
|
+
# 首帧之后帧间隔超时兜底,防「服务端不再吐 token」永久阻塞
|
|
932
|
+
timeout = _STREAM_FRAME_INTERVAL_TIMEOUT if self._got_first else None
|
|
933
|
+
try:
|
|
934
|
+
item = self._q.get(timeout=timeout)
|
|
935
|
+
except queue.Empty as exc:
|
|
936
|
+
self.close()
|
|
937
|
+
raise AITimeoutError(
|
|
938
|
+
f"流式帧间隔超过 {_STREAM_FRAME_INTERVAL_TIMEOUT:g}s 未收到新帧"
|
|
939
|
+
) from exc
|
|
940
|
+
if item is _SENTINEL:
|
|
941
|
+
self._exhausted = True
|
|
942
|
+
self.close()
|
|
943
|
+
raise StopIteration
|
|
944
|
+
if isinstance(item, BaseException):
|
|
945
|
+
self.close()
|
|
946
|
+
raise item
|
|
947
|
+
self._got_first = True
|
|
948
|
+
return str(item)
|
|
949
|
+
|
|
950
|
+
# ---------- 收尾统计(只读) ----------
|
|
951
|
+
|
|
952
|
+
@property
|
|
953
|
+
def reasoning_chars(self) -> int:
|
|
954
|
+
"""推理(思考)内容字符数;非思考模型恒为 0。"""
|
|
955
|
+
return self._stats.reasoning_chars
|
|
956
|
+
|
|
957
|
+
@property
|
|
958
|
+
def content_chars(self) -> int:
|
|
959
|
+
"""已产出正文字符数(与本迭代器产出的字符数严格相等)。"""
|
|
960
|
+
return self._stats.content_chars
|
|
961
|
+
|
|
962
|
+
@property
|
|
963
|
+
def finish_reason(self) -> str | None:
|
|
964
|
+
"""结束原因:``stop`` / ``length`` / ``content_filter`` / …;未给出为 None。"""
|
|
965
|
+
return self._stats.finish_reason
|
|
966
|
+
|
|
967
|
+
@property
|
|
968
|
+
def truncated(self) -> bool:
|
|
969
|
+
"""是否因输出长度上限被截断(``finish_reason`` 为 length / max_tokens)。"""
|
|
970
|
+
return self._stats.truncated
|
|
971
|
+
|
|
972
|
+
@property
|
|
973
|
+
def thinking_applied(self) -> bool | None:
|
|
974
|
+
"""思考开关是否生效:None 未要求干预 / True 已注入 / False 被忽略。"""
|
|
975
|
+
return self._stats.thinking_applied
|
|
976
|
+
|
|
977
|
+
@property
|
|
978
|
+
def json_mode_applied(self) -> bool | None:
|
|
979
|
+
"""结构化输出是否生效:None 未要求注入 / True 已注入 / False 被忽略。"""
|
|
980
|
+
return self._stats.json_mode_applied
|
|
981
|
+
|
|
982
|
+
@property
|
|
983
|
+
def requested_max_tokens(self) -> int | None:
|
|
984
|
+
"""本次实际下发的输出上限(调用方 → 档案回落);未发请求为 None。"""
|
|
985
|
+
return self._stats.requested_max_tokens
|
|
986
|
+
|
|
987
|
+
@property
|
|
988
|
+
def empty_kind(self) -> str | None:
|
|
989
|
+
"""正文为空时的诊断分级;**仅在流正常结束后可读,其余时刻为 None**。
|
|
990
|
+
|
|
991
|
+
取值同 ``StreamStats.empty_kind``(``budget_exhausted`` /
|
|
992
|
+
``truncated_empty`` / ``reasoning_only`` / ``no_finish`` /
|
|
993
|
+
``stopped_empty``;正文非空为 None)。
|
|
994
|
+
|
|
995
|
+
迭代中不给中间结论:此时正文尚未产出,``content_chars`` 恒为 0,
|
|
996
|
+
直接转发会把「还没出正文」误报成 ``no_finish``;异常 / 中途 ``break``
|
|
997
|
+
同理不置收尾标志。
|
|
998
|
+
"""
|
|
999
|
+
if not self._exhausted:
|
|
1000
|
+
return None
|
|
1001
|
+
return self._stats.empty_kind
|
|
1002
|
+
|
|
1003
|
+
|
|
1004
|
+
async def _delayed_close(services: list[Service]) -> None:
|
|
1005
|
+
"""延迟回收旧 service:给在途调用留宽限期,到期统一关闭(须在 loop 内)。"""
|
|
1006
|
+
await asyncio.sleep(_DELAYED_CLOSE_SECONDS)
|
|
1007
|
+
for service in services:
|
|
1008
|
+
try:
|
|
1009
|
+
await service.close()
|
|
1010
|
+
except Exception as exc: # noqa: BLE001 - 回收阶段失败不影响主流程
|
|
1011
|
+
logger.warning(f"延迟回收旧 AI service 失败: {exc}")
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
@dataclass(frozen=True)
|
|
1015
|
+
class ReloadResult:
|
|
1016
|
+
"""配置重载结果摘要。
|
|
1017
|
+
|
|
1018
|
+
Attributes:
|
|
1019
|
+
types: 实际重建的模型类型(全量重载为 4 类型的排序列表)
|
|
1020
|
+
embedding_model_changed: embedding 默认档案的 ``model`` 或
|
|
1021
|
+
``dimension`` 是否变化。``dimension`` 是**目标向量维度**
|
|
1022
|
+
(`EmbeddingService` 保证输出等于它),故它变化会直接改变
|
|
1023
|
+
输出维度;已建向量库的维度与存量向量不会随重载迁移,
|
|
1024
|
+
变化时调用方应评估是否需要重新入库(kit 只上报不拦截——
|
|
1025
|
+
新建库场景下更换 embedding 是合法操作)
|
|
1026
|
+
previous_embedding_model: 重载前 embedding 默认档案的 model
|
|
1027
|
+
current_embedding_model: 重载后 embedding 默认档案的 model
|
|
1028
|
+
"""
|
|
1029
|
+
|
|
1030
|
+
types: list[str]
|
|
1031
|
+
embedding_model_changed: bool = False
|
|
1032
|
+
previous_embedding_model: str | None = None
|
|
1033
|
+
current_embedding_model: str | None = None
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def _embedding_snapshot() -> tuple[str, int | None] | None:
|
|
1037
|
+
"""embedding 默认档案的 ``(model, dimension)`` 快照;类型未配置返回 None。"""
|
|
1038
|
+
try:
|
|
1039
|
+
config = get_model_registry().get_profile("embedding")
|
|
1040
|
+
except AIConfigError:
|
|
1041
|
+
return None
|
|
1042
|
+
return (config.model, config.dimension)
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
def reload_config(
|
|
1046
|
+
*,
|
|
1047
|
+
model_type: str | None = None,
|
|
1048
|
+
reload_file: bool = False,
|
|
1049
|
+
probe: bool = True,
|
|
1050
|
+
immediate: bool = False,
|
|
1051
|
+
) -> ReloadResult:
|
|
1052
|
+
"""配置热重载:失效候选链与常驻服务,使新配置对后续调用立即生效。
|
|
1053
|
+
|
|
1054
|
+
典型用法(``os.environ`` 的更新是调用方职责;kit 不重解析 ``.env``,
|
|
1055
|
+
``load_dotenv(override=False)`` 的「真实环境变量优先」语义不受影响)::
|
|
1056
|
+
|
|
1057
|
+
os.environ["SOME_API_KEY"] = new_key # 调用方:注入新密钥
|
|
1058
|
+
reload_config() # kit:失效候选链与服务
|
|
1059
|
+
|
|
1060
|
+
Args:
|
|
1061
|
+
model_type: None 重建全部 4 类型;指定则仅重建该类型(其余类型的
|
|
1062
|
+
候选链、常驻服务、在途调用与探测状态均不受影响)
|
|
1063
|
+
reload_file: True 时额外重读 ``ai_models.json``(增删档案 / 改
|
|
1064
|
+
model 等静态配置变更时用);False 只重读 ``os.environ`` 密钥,
|
|
1065
|
+
**不会**重载 JSON
|
|
1066
|
+
probe: True 时重置预热探测,下次取服务按失效类型重新投递;
|
|
1067
|
+
False 时不强制重置(沿用既有探测机制,已投递过则不再投递)
|
|
1068
|
+
immediate: True 时旧 service 立即关闭;默认延迟 60 秒统一回收,
|
|
1069
|
+
在途调用(含流式)可在宽限期内继续
|
|
1070
|
+
|
|
1071
|
+
Returns:
|
|
1072
|
+
:class:`ReloadResult`:重建类型列表与 embedding 默认档案变更告警。
|
|
1073
|
+
|
|
1074
|
+
Raises:
|
|
1075
|
+
AIConfigError: ``model_type`` 非法;或 ``reload_file=True`` 且档案
|
|
1076
|
+
文件缺失 / 非法(此时旧配置继续运行,已建链与服务不受影响)。
|
|
1077
|
+
RuntimeError: 在后台事件循环线程内调用(同步门面不支持重入)。
|
|
1078
|
+
|
|
1079
|
+
契约与限制:
|
|
1080
|
+
|
|
1081
|
+
- **核心保证**:本函数返回后,新调用必定使用新配置;
|
|
1082
|
+
- 关闭语义:默认旧 service 延迟 60s 回收,宽限期后仍未完成的
|
|
1083
|
+
在途调用可能被关闭;``immediate=True`` 返回即关闭,在途调用
|
|
1084
|
+
按关闭语义报错,调用方自担;
|
|
1085
|
+
- 进程级操作:多 worker 部署仅当前进程生效;
|
|
1086
|
+
- 副作用:清空 kit 侧 ``get_settings`` 缓存(LOG / DB 等全部
|
|
1087
|
+
配置随下次访问重读,值未变则行为不变)。
|
|
1088
|
+
"""
|
|
1089
|
+
if model_type is not None and model_type not in _SERVICE_TYPES:
|
|
1090
|
+
raise AIConfigError(
|
|
1091
|
+
f"未知模型类型: {model_type},支持: {', '.join(sorted(_SERVICE_TYPES))}"
|
|
1092
|
+
)
|
|
1093
|
+
# 1. 先清 kit 配置缓存:后续 registry.load / chain 解析均读新 env 值
|
|
1094
|
+
get_settings.cache_clear()
|
|
1095
|
+
# 2. 快照旧 embedding(此刻 registry 未动;类型未配置容错为 None)
|
|
1096
|
+
previous = _embedding_snapshot()
|
|
1097
|
+
# 3. 重读档案文件:校验失败即抛 AIConfigError 中止,_config 未被覆盖,
|
|
1098
|
+
# 已建链与服务不受影响(旧配置继续运行,重载整体原子)
|
|
1099
|
+
if reload_file:
|
|
1100
|
+
get_model_registry().load()
|
|
1101
|
+
# 4. 后台 loop 内摘链、清常驻服务并调度旧 service 关闭
|
|
1102
|
+
types = sorted(_SERVICE_TYPES) if model_type is None else [model_type]
|
|
1103
|
+
_background.invalidate(frozenset(types), probe=probe, immediate=immediate)
|
|
1104
|
+
# 4.1 重置告警去重:修正档案后同一原因应能再次提示(去重键不含配置指纹)
|
|
1105
|
+
reset_warned()
|
|
1106
|
+
# 5. 快照新 embedding,判定变更(model 或 dimension 任一变化即告警)
|
|
1107
|
+
current = _embedding_snapshot()
|
|
1108
|
+
changed = previous is not None and current is not None and previous != current
|
|
1109
|
+
if changed:
|
|
1110
|
+
logger.warning(
|
|
1111
|
+
f"embedding 默认档案已变化: {previous} → {current},"
|
|
1112
|
+
"已建向量库维度与存量向量不会自动迁移,调用方应评估是否需重新入库"
|
|
1113
|
+
)
|
|
1114
|
+
return ReloadResult(
|
|
1115
|
+
types=types,
|
|
1116
|
+
embedding_model_changed=changed,
|
|
1117
|
+
previous_embedding_model=previous[0] if previous is not None else None,
|
|
1118
|
+
current_embedding_model=current[0] if current is not None else None,
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def shutdown() -> None:
|
|
1123
|
+
"""停止后台事件循环并释放常驻 AI 服务(应用退出前调用,幂等)。"""
|
|
1124
|
+
_background.shutdown()
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
__all__ = ["AI", "ReloadResult", "reload_config", "run_sync", "shutdown"]
|