llm-engine-kitty 0.1.0.dev0__py3-none-any.whl

llm_engine/schemas.py

@@ -0,0 +1,251 @@
+# llm_engine/schemas.py
+
+import asyncio
+import threading
+import time
+
+from enum import Enum
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+from typing import Any, Callable, Dict, List, Optional
+
+
+class MessageRole(str, Enum):
+    USER = "user"
+    ASSISTANT = "assistant"
+    SYSTEM = "system"
+
+
+class Message(BaseModel):
+    role: str = Field(description="消息角色")
+    content: str = Field(description="消息内容")
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "role": self.role,
+            "content": self.content,
+        }
+
+    def __repr__(self) -> str:
+        _preview: str
+        if len(self.content) > 48:
+            _preview = f"{self.content[:16]}......{self.content[-16:]}"
+        else:
+            _preview = self.content
+        return f"Message(role={self.role!r}, content={_preview!r})"
+
+
+class InferenceParameters(BaseModel):
+    model_config = ConfigDict(
+        extra="allow",
+        populate_by_name=True,
+        str_strip_whitespace=True,
+        validate_assignment=True,
+        use_enum_values=True,
+        arbitrary_types_allowed=True,
+    )
+
+    max_tokens: Optional[int] = Field(default=None)
+    temperature: Optional[float] = Field(default=None)
+    top_p: Optional[float] = Field(default=None)
+    top_k: Optional[int] = Field(default=None)
+    frequency_penalty: Optional[float] = Field(default=None)
+    n: Optional[int] = Field(default=None)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self.model_dump(exclude_none=True)
+
+
+class InferenceRequest(BaseModel):
+    model_name: Optional[str] = Field(default=None)
+    api_key: Optional[str] = Field(default=None)
+    messages: List[Message]
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    inference_parameters: Optional[InferenceParameters] = Field(default=None)
+    stream: Optional[bool] = Field(default=None)
+    timeout: Optional[float] = Field(default=None)
+
+    extra_headers: Dict[str, str] = Field(default_factory=dict)
+    extra_payload: Dict[str, Any] = Field(default_factory=dict)
+
+
+class PreparedRequest(BaseModel):
+    model_name: str
+    url: str
+    urls: list[str] = Field(default_factory=list)
+    headers: Dict[str, str]
+    payload: Dict[str, Any]
+
+
+class TaskStatus(str, Enum):
+    # —— 非终态 ——
+    SUBMITTED = "submitted"  # 已提交，等待验证（如 model 解析、registry 查找）
+    ACCEPTED = "accepted"  # 验证通过，待入队等待 sem
+    PENDING = "pending"  # 已入队，正在等待 sem 资源
+    RUNNING = "running"  # 已抢到 sem，正在执行
+
+    # —— 终态 ——
+    SUCCESS = "success"
+    FAILED = "failed"  # 执行阶段失败（HTTP / 解析 / 网络等非超时异常）
+    TIMEOUT = "timeout"  # 执行阶段超时（task_timeout 触发）
+    CANCELLED = "cancelled"
+    REJECTED = "rejected"  # 验证阶段失败（非法任务，未进入执行阶段）
+
+
+class ModelOutput(BaseModel):
+    role: Optional[str] = None
+    reasoning: Optional[str] = None
+    content: str
+    finish_reason: Optional[str] = None
+    usage: Optional[Dict[str, Any]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self.model_dump(exclude_none=False)
+
+
+class InferenceRequestResult(BaseModel):
+
+    success: bool
+
+    task_id: str
+    request: InferenceRequest
+    model_output: Optional[ModelOutput] = None
+    error_message: Optional[str] = None
+    duration: Optional[float] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self.model_dump(exclude_none=False)
+
+
+class TaskHandle(BaseModel):
+    # 核心配置
+    model_config = ConfigDict(arbitrary_types_allowed=True, validate_assignment=True)
+
+    # 公共数据字段
+    task_id: str
+    request: InferenceRequest
+    status: TaskStatus = TaskStatus.SUBMITTED
+    result: Optional[InferenceRequestResult] = None
+    error: Optional[str] = None
+    error_exception: Optional[Exception] = None
+    tag: Optional[str] = None
+    submit_time: float = Field(default_factory=time.time)
+    start_time: Optional[float] = None
+    end_time: Optional[float] = None
+    task_timeout: Optional[float] = None
+    persist: bool = False
+
+    # 流式内容
+    partial_response: str = ""
+    partial_reasoning: str = ""
+
+    # 运行时对象：不序列化、不在构造参数中
+    _asyncio_task: Optional[asyncio.Task] = PrivateAttr(default=None)
+    _done_event: Optional[asyncio.Event] = PrivateAttr(default=None)
+    _done_threading_event: Optional[threading.Event] = PrivateAttr(default=None)
+
+    # 回调
+    _on_token: Optional[Callable] = PrivateAttr(default=None)
+    _on_success: Optional[Callable] = PrivateAttr(default=None)
+    _on_failure: Optional[Callable] = PrivateAttr(default=None)
+    _on_retry: Optional[Callable] = PrivateAttr(default=None)
+
+    @property
+    def duration(self) -> Optional[float]:
+        if self.start_time is None:
+            return None
+        end = self.end_time or time.time()
+        return end - self.start_time
+
+    def is_finished(self) -> bool:
+        return self.status in (
+            TaskStatus.SUCCESS,
+            TaskStatus.FAILED,
+            TaskStatus.TIMEOUT,
+            TaskStatus.CANCELLED,
+            TaskStatus.REJECTED,
+        )
+
+    def is_timeout(self) -> bool:
+        if self.task_timeout is None:
+            return False
+        return (time.time() - self.submit_time) > self.task_timeout
+
+
+class EngineStats(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+    # 任务状态计数
+    total_submitted: int = Field(0, ge=0)
+    total_pending: int = Field(0, ge=0)
+    total_running: int = Field(0, ge=0)
+    total_done: int = Field(0, ge=0)
+    total_failed: int = Field(0, ge=0)
+    total_cancelled: int = Field(0, ge=0)
+
+    # 资源消耗统计
+    total_input_tokens: int = Field(0, ge=0)
+    total_output_tokens: int = Field(0, ge=0)
+    total_tokens: int = Field(0, ge=0)
+    total_duration: float = Field(0.0, ge=0.0)
+    estimated_cost: float = Field(0.0, ge=0.0)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self.model_dump()
+
+    @property
+    def success_rate(self) -> float:
+        """计算成功率"""
+        if self.total_done + self.total_failed == 0:
+            return 0.0
+        return self.total_done / (self.total_done + self.total_failed)
+
+
+class ChunkDelta(BaseModel):
+    role: Optional[str] = None
+    content: Optional[str] = None
+    reasoning_content: Optional[str] = None
+
+
+class ChunkChoice(BaseModel):
+    index: int
+    delta: ChunkDelta
+    finish_reason: Optional[str] = None
+    logprobs: Optional[Any] = None
+
+
+class TokenUsage(BaseModel):
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    completion_tokens_details: Optional[Dict[str, Any]] = None
+
+
+class ChatCompletionChunk(BaseModel):
+    id: str
+    object: str
+    created: int
+    model: str
+    choices: List[ChunkChoice]
+    usage: Optional[TokenUsage] = None
+    system_fingerprint: Optional[str] = None
+
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+    reasoning_content: Optional[str] = None
+
+
+class ChatCompletionChoice(BaseModel):
+    index: int
+    message: ChatMessage
+    finish_reason: Optional[str] = None
+
+
+class ChatCompletionResponse(BaseModel):
+    id: str
+    object: str
+    created: int
+    model: str
+    choices: List[ChatCompletionChoice]
+    usage: Optional[TokenUsage] = None

llm_engine/utils.py

@@ -0,0 +1,34 @@
+# llm_engine/utils.py
+
+import time
+import uuid
+
+from typing import Optional
+
+import kitty_logger
+
+logger = kitty_logger.getLogger(__name__)
+
+
+def get_timestamp() -> str:
+    timestamp_ns = time.time_ns()
+    return f"{timestamp_ns:020d}"
+
+
+def get_timestamp_h() -> str:
+
+    timestamp_ns = time.time_ns()
+
+    seconds, nanoseconds = divmod(timestamp_ns, 1_000_000_000)
+
+    t = time.localtime(seconds)
+
+    date_time_str = time.strftime("%Y_%m_%d_%H_%M_%S", t)
+
+    return f"{date_time_str}_{nanoseconds:09d}"
+
+
+def gen_unique_id(prefix: Optional[str] = None) -> str:
+    uid = uuid.uuid4()
+    ts = get_timestamp_h()
+    return f"{prefix}_{ts}_{uid}" if prefix else f"{ts}_{uid}"

llm_engine_kitty-0.1.0.dev0.dist-info/METADATA

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: llm-engine-kitty
+Version: 0.1.0.dev0
+Summary: A lightweight LLM inference engine supporting OpenAI-compatible APIs.
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.0
+Requires-Dist: httpx>=0.28
+Requires-Dist: tqdm>=4.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: msgpack>=1.0
+Requires-Dist: kitty-logger>=0.2.0.dev3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: black; extra == "dev"
+Requires-Dist: python-dotenv; extra == "dev"

llm_engine_kitty-0.1.0.dev0.dist-info/RECORD

@@ -0,0 +1,18 @@
+llm_engine/__init__.py,sha256=-7ezeNIQ1gDo_xyhZ2rtf1eDh10_F78x4aRujbXkdog,1396
+llm_engine/engine.py,sha256=ZHY7oisg4BGGWDAu0oqf2x-lQaC_8umPzEPMOqBmaNo,34125
+llm_engine/general_engine.py,sha256=AHlCCRvMO5wzGlSGO3dJS-YPlMVXebDjf3VPjKJFbs4,24398
+llm_engine/model_config.py,sha256=1THnQ5HmDxBOABRLIvVo-f-yTnGeEUsQSCwHQbraV14,3800
+llm_engine/schemas.py,sha256=nxfClEEkwGjL4am42VkGQ6uYVj-vCUqzbFqjvYjLn-w,7366
+llm_engine/utils.py,sha256=4z99QOzqW7qo5H_S-TQhuc9biTZH80lBrA0YzJE8e6M,681
+llm_engine/kitty/__init__.py,sha256=9b1ESBxdjmwZCAd1HgP7XyqEm7fHZIuDWNkaY7eXGRg,236
+llm_engine/kitty/__main__.py,sha256=f4Qufj3ZCh6s5ajuIKwV9bl4RPhWbDm_rOChsf1HbgQ,1244
+llm_engine/kitty/client.py,sha256=rO-7AESgO4wchjRE4CXiy4Norrs8S-BIuzCYxzsP9Q0,22652
+llm_engine/kitty/config.py,sha256=BVtnEwDC4MHcaHto3tioGlO_tb8i6W2TbupnbdRrl-4,3403
+llm_engine/kitty/engine.py,sha256=mtTKMfROSNDR0PD54MQI_tHR6bJO_aodEHFQIFvFvCs,50564
+llm_engine/kitty/protocol.py,sha256=m-DlhxwqUVENPZg5BvfeR84BuLSeCy-l_33riq7oTlM,7331
+llm_engine/kitty/schemas.py,sha256=GAHBLEUCtnTf2heiYIEI8IIWKtwBrdiQPsZ7kOINjsA,3029
+llm_engine/kitty/server.py,sha256=Hnrasl_0sDTgb-_vrq9y0DE7QPfFFYX4WHUbQvFzSTU,18223
+llm_engine_kitty-0.1.0.dev0.dist-info/METADATA,sha256=BYz6YIXkRLOUwvkzlEj-VEm5xIZREXf2S2VZRob9tWY,489
+llm_engine_kitty-0.1.0.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+llm_engine_kitty-0.1.0.dev0.dist-info/top_level.txt,sha256=az8036N8tKuDqgMteGVC6Q86IC92YXspiWgo440tARo,11
+llm_engine_kitty-0.1.0.dev0.dist-info/RECORD,,

llm_engine_kitty-0.1.0.dev0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

llm_engine_kitty-0.1.0.dev0.dist-info/top_level.txt

@@ -0,0 +1 @@
+llm_engine