sdpy-kit 0.1.0__py3-none-any.whl

kit/common_kit.py

@@ -0,0 +1,221 @@
+"""
+Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Project: sdpy
+Author: Clark Chang
+"""
+
+# date: 2026-08-22
+"""公共工具类，提供雪花算法 ID 生成和系统时间格式化功能
+
+重构说明（破坏性内部、外部零风险）：
+- 将原先 `_initialize` / `configure` / `_INITIALIZED` 三套冗余的初始化
+  逻辑收敛为唯一的 `configure` 入口 + 惰性 `_ensure_initialized`。
+- 用 `_DATACENTER_ID is None` 语义替代 `_INITIALIZED` 布尔标记，状态更清晰。
+- 公开方法名与签名不变。
+"""
+
+import threading
+import time
+from datetime import datetime
+from random import randint
+
+
+class CommonKit:
+    """公共工具类，提供分布式唯一 ID 生成和系统时间格式化功能
+
+    雪花算法 ID 结构（共 63 位，符号位为 0）：
+    - timestamp(41位)：毫秒级时间戳，约可用 69 年
+    - datacenter_id(5位)：数据中心 ID，有效范围 1-31
+    - machine_id(5位)：机器 ID，有效范围 1-31
+    - sequence(12位)：每毫秒内序列号，最大值 4095
+
+    支持多线程并发调用，保证 ID 唯一性。
+    """
+
+    _TIMESTAMP_SHIFT: int = 22
+    _DATACENTER_ID_SHIFT: int = 17
+    _MACHINE_ID_SHIFT: int = 12
+
+    _MAX_DATACENTER_ID: int = 31
+    _MAX_MACHINE_ID: int = 31
+    _MAX_SEQUENCE: int = 4095
+
+    _EPOCH: int = 1577836800000
+
+    # None 表示尚未初始化（未调用 configure）
+    _DATACENTER_ID: int | None = None
+    _MACHINE_ID: int | None = None
+
+    _lock: threading.Lock = threading.Lock()
+
+    _last_timestamp: int = -1
+    _sequence: int = 0
+
+    @classmethod
+    def configure(cls, datacenter_id: int, machine_id: int) -> None:
+        """配置雪花算法的数据中心 ID 和机器 ID
+
+        Args:
+            datacenter_id: 数据中心 ID，有效范围 1-31
+            machine_id: 机器 ID，有效范围 1-31
+
+        Raises:
+            ValueError: 参数超出有效范围时抛出
+        """
+        if not (1 <= datacenter_id <= cls._MAX_DATACENTER_ID):
+            raise ValueError(
+                f"数据中心 ID 必须在 1-{cls._MAX_DATACENTER_ID} 之间，"
+                f"当前值: {datacenter_id}"
+            )
+        if not (1 <= machine_id <= cls._MAX_MACHINE_ID):
+            raise ValueError(
+                f"机器 ID 必须在 1-{cls._MAX_MACHINE_ID} 之间，当前值: {machine_id}"
+            )
+        with cls._lock:
+            cls._DATACENTER_ID = datacenter_id
+            cls._MACHINE_ID = machine_id
+
+    @classmethod
+    def _ensure_initialized(cls) -> None:
+        """惰性初始化：首次使用时采用默认配置 (1, 1)"""
+        if cls._DATACENTER_ID is None:
+            cls.configure(1, 1)
+
+    @classmethod
+    def _get_current_timestamp(cls) -> int:
+        """获取当前时间戳（毫秒）"""
+        return int(time.time() * 1000)
+
+    @classmethod
+    def _wait_for_next_millis(cls, last_timestamp: int) -> int:
+        """等待直到进入下一毫秒"""
+        timestamp = cls._get_current_timestamp()
+        while timestamp <= last_timestamp:
+            timestamp = cls._get_current_timestamp()
+        return timestamp
+
+    @classmethod
+    def generate_snowflake_id(
+        cls, datacenter_id: int | None = None, machine_id: int | None = None
+    ) -> int:
+        """生成雪花算法 ID（数值类型）
+
+        Args:
+            datacenter_id: 数据中心 ID（可选），有效范围 1-31，默认使用配置值
+            machine_id: 机器 ID（可选），有效范围 1-31，默认使用配置值
+
+        Returns:
+            int: 63 位雪花 ID 数值
+
+        Raises:
+            ValueError: 参数超出有效范围或时钟回拨时抛出
+        """
+        if datacenter_id is None or machine_id is None:
+            cls._ensure_initialized()
+            assert cls._DATACENTER_ID is not None
+            assert cls._MACHINE_ID is not None
+            if datacenter_id is None:
+                datacenter_id = cls._DATACENTER_ID
+            if machine_id is None:
+                machine_id = cls._MACHINE_ID
+
+        if not (1 <= datacenter_id <= cls._MAX_DATACENTER_ID):
+            raise ValueError(f"数据中心 ID 必须在 1-{cls._MAX_DATACENTER_ID} 之间")
+        if not (1 <= machine_id <= cls._MAX_MACHINE_ID):
+            raise ValueError(f"机器 ID 必须在 1-{cls._MAX_MACHINE_ID} 之间")
+
+        with cls._lock:
+            timestamp = cls._get_current_timestamp()
+
+            if timestamp < cls._last_timestamp:
+                raise ValueError(
+                    f"时钟回拨检测：当前时间戳({timestamp})小于上次使用时间戳"
+                    f"({cls._last_timestamp})，无法生成 ID，请检查系统时间"
+                )
+
+            if timestamp == cls._last_timestamp:
+                cls._sequence = (cls._sequence + 1) & cls._MAX_SEQUENCE
+                if cls._sequence == 0:
+                    timestamp = cls._wait_for_next_millis(cls._last_timestamp)
+            else:
+                cls._sequence = 0
+
+            cls._last_timestamp = timestamp
+
+            return (
+                ((timestamp - cls._EPOCH) << cls._TIMESTAMP_SHIFT)
+                | (datacenter_id << cls._DATACENTER_ID_SHIFT)
+                | (machine_id << cls._MACHINE_ID_SHIFT)
+                | cls._sequence
+            )
+
+    @classmethod
+    def generate_snow_id(
+        cls, datacenter_id: int | None = None, machine_id: int | None = None
+    ) -> str:
+        """生成雪花算法 ID（字符串类型）
+
+        Args:
+            datacenter_id: 数据中心 ID（可选），有效范围 1-31，默认使用配置值
+            machine_id: 机器 ID（可选），有效范围 1-31，默认使用配置值
+
+        Returns:
+            str: 雪花 ID 字符串形式
+        """
+        return str(cls.generate_snowflake_id(datacenter_id, machine_id))
+
+    @staticmethod
+    def generate_timestamp_id() -> str:
+        """生成基于当前时间的唯一 ID（20 位）
+
+        格式: YYYYMMDDHHMMSSfffRRR，后 3 位为随机数确保同毫秒内唯一性
+        """
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f")[:-3]
+        random_suffix = randint(0, 999)
+        return f"{timestamp}{random_suffix:03d}"
+
+    @staticmethod
+    def format_current_time(format_pattern: str = "yyyyMMdd") -> str:
+        """格式化当前系统时间
+
+        支持格式: yyyyMMdd, yyyyMMddhhmm, yyyyMMddhhmmss,
+                 yyyy-MM-dd, yyyy-MM-dd hh:mm, yyyy-MM-dd hh:mm:ss
+        """
+        format_map = {
+            "yyyyMMdd": "%Y%m%d",
+            "yyyyMMddhhmm": "%Y%m%d%H%M",
+            "yyyyMMddhhmmss": "%Y%m%d%H%M%S",
+            "yyyy-MM-dd": "%Y-%m-%d",
+            "yyyy-MM-dd hh:mm": "%Y-%m-%d %H:%M",
+            "yyyy-MM-dd hh:mm:ss": "%Y-%m-%d %H:%M:%S",
+        }
+        if format_pattern not in format_map:
+            raise ValueError(
+                f"不支持的格式模式: {format_pattern}，支持的格式有: "
+                f"{', '.join(format_map.keys())}"
+            )
+
+        try:
+            return time.strftime(format_map[format_pattern])
+        except Exception as e:
+            raise RuntimeError(f"获取系统时间失败: {e}")
+
+    @classmethod
+    def reset(cls) -> None:
+        """重置 ID 生成器状态"""
+        with cls._lock:
+            cls._last_timestamp = -1
+            cls._sequence = 0

kit/config/__init__.py

@@ -0,0 +1,44 @@
+"""
+Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Project: sdpy
+Author: Clark Chang
+"""
+
+# date: 2026-09-08
+"""Kit 配置子包：统一导出配置与日志入口"""
+
+from kit.config.kit_config import Settings, get_settings
+from kit.config.log_config import (
+    critical,
+    debug,
+    error,
+    get_default_logger,
+    info,
+    setup_logger,
+    warning,
+)
+
+__all__ = [
+    "Settings",
+    "get_settings",
+    "setup_logger",
+    "get_default_logger",
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "critical",
+]

kit/config/kit_config.py

@@ -0,0 +1,147 @@
+"""
+Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Project: sdpy
+Author: Clark Chang
+"""
+
+# date: 2025-04-19
+# source: 迁移自 config/sys_config.py（kit 自包含配置，修改须与源文件评估双向同步）
+"""Kit 统一配置模块"""
+
+from functools import lru_cache
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    APP_NAME: str = Field(default="SDPY App")
+    VERSION: str = Field(default="0.1.0")
+    DEBUG: bool = Field(default=True)
+    HOST: str = Field(default="0.0.0.0")
+    PORT: int = Field(default=8000)
+
+    LOG_LEVEL: str = Field(default="INFO")
+    LOG_TO_CONSOLE: bool = Field(default=True)
+    LOG_TO_FILE: bool = Field(default=False)
+    LOG_DIR: str = Field(default="./logs")
+    LOG_FILE: str = Field(default="app.log")
+
+    REDIS_HOST: str = Field(default="localhost")
+    REDIS_PORT: int = Field(default=6379)
+    REDIS_DB: int = Field(default=0)
+    REDIS_PASSWORD: str | None = Field(default=None)
+
+    # 数据库统一配置（PostgreSQL / MySQL 共用）
+    # 切换数据库只需修改 DEFAULT_DB_TYPE，无需修改连接参数
+    DEFAULT_DB_TYPE: str = Field(default="postgresql")
+
+    DB_HOST: str = Field(default="localhost")
+    # PostgreSQL 专用
+    DB_PORT: int = Field(default=5432)
+    DB_DATABASE: str = Field(default="postgres")
+    DB_USER: str = Field(default="admin")
+    DB_PASSWORD: str = Field(default="1qaz@WSX3edc")
+
+    # MySQL 专用覆盖（当端口/用户/库名与 PG 不同时使用）
+    MYSQL_PORT: int = Field(default=3306)
+    MYSQL_DATABASE: str = Field(default="mydb")
+    MYSQL_USER: str = Field(default="root")
+    MYSQL_CHARSET: str = Field(default="utf8mb4")
+
+    # SQLite 专用（切换到 SQLite 时取消注释）
+    SQLITE_DB_PATH: str = Field(default="./data/sqlite.db")
+
+    # zvec 向量库配置（本地嵌入式向量库，独立于 kit.db）
+    ZVEC_DATA_DIR: str = Field(
+        default="./data/zvec", description="zvec Collection 根目录"
+    )
+    ZVEC_QUERY_THREADS: int | None = Field(
+        default=None, description="查询线程数，None 自适应"
+    )
+    ZVEC_OPTIMIZE_THREADS: int | None = Field(
+        default=None, description="索引构建线程数，None 自适应"
+    )
+    ZVEC_MEMORY_LIMIT_MB: int | None = Field(
+        default=None, description="内存软上限(MB)，None 自适应"
+    )
+    ZVEC_DEFAULT_METRIC: str = Field(default="cosine", description="默认向量距离度量")
+    ZVEC_LOG_DIR: str = Field(
+        default="./logs/zvec", description="zvec 引擎独立日志目录"
+    )
+    ZVEC_LOG_LEVEL: str = Field(
+        default="warn", description="zvec 引擎日志级别(debug/info/warn/error/fatal)"
+    )
+
+    FILE_STORAGE_ROOT: str = Field(default="./data")
+
+    # AI 模型档案配置文件路径（chat/embedding/rerank/image 四类型多厂商档案）
+    AI_MODELS_CONFIG_PATH: str = Field(
+        default="./config/ai_models.json", description="AI 模型档案配置文件路径"
+    )
+
+    # AI 候选链优先级（可选，逗号分隔档案名）：设置后覆盖 JSON default 顺序；
+    # JSON 中不存在的档案名会被丢弃并告警
+    AI_CHAT_PROFILES: str | None = Field(
+        default=None, description="chat 类型候选链优先级（逗号分隔档案名）"
+    )
+    AI_EMBEDDING_PROFILES: str | None = Field(
+        default=None, description="embedding 类型候选链优先级（逗号分隔档案名）"
+    )
+    AI_RERANK_PROFILES: str | None = Field(
+        default=None, description="rerank 类型候选链优先级（逗号分隔档案名）"
+    )
+    AI_IMAGE_PROFILES: str | None = Field(
+        default=None, description="image 类型候选链优先级（逗号分隔档案名）"
+    )
+
+    # MCP 客户端配置（通用 MCP 服务器档案，schema 见 config/mcp_servers.json）
+    MCP_SERVERS_CONFIG_PATH: str = Field(
+        default="./config/mcp_servers.json", description="MCP 服务器档案文件路径"
+    )
+    MCP_CALL_TIMEOUT: int = Field(
+        default=60,
+        ge=1,
+        le=600,
+        description="MCP 调用默认超时（秒），档案未指定 timeout 时使用",
+    )
+
+    # MinerU 精准解析 API 配置（https://mineru.net/apiManage/docs）
+    MINERU_API_KEY: str | None = Field(
+        default=None, description="MinerU API Key（API 管理页面自建）"
+    )
+    MINERU_BASE_API_URL: str = Field(
+        default="https://mineru.net", description="MinerU API 基础地址"
+    )
+    MINERU_TIMEOUT: int = Field(
+        default=60, ge=1, le=600, description="单次 HTTP 请求超时（秒）"
+    )
+    MINERU_POLL_INTERVAL: float = Field(
+        default=2.0, ge=0.5, description="任务轮询间隔（秒）"
+    )
+    MINERU_MAX_WAIT: int = Field(
+        default=600, ge=10, description="任务轮询总超时（秒），超时抛 MinerUError"
+    )
+
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+        extra = "ignore"
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    return Settings()

kit/config/log_config.py

@@ -0,0 +1,134 @@
+"""
+Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Project: sdpy
+Author: Clark Chang
+"""
+
+# date: 2025-04-19
+# source: 迁移自 config/logger_config.py（kit 自包含日志，修改须与源文件评估双向同步）
+"""Kit 统一日志配置模块"""
+
+import logging
+import os
+import sys
+from logging.handlers import RotatingFileHandler
+
+from kit.config.kit_config import get_settings
+
+settings = get_settings()
+
+DEFAULT_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+DEFAULT_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+LOG_LEVEL_MAPPING = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "WARN": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "FATAL": logging.CRITICAL,
+}
+
+
+def ensure_log_dir(log_dir: str) -> None:
+    """确保日志目录存在"""
+    os.makedirs(log_dir, exist_ok=True)
+
+
+def setup_logger(
+    name: str | None = None,
+    log_level: str | None = None,
+    log_file: str | None = None,
+    log_format: str = DEFAULT_LOG_FORMAT,
+    date_format: str = DEFAULT_DATE_FORMAT,
+    max_bytes: int = 5 * 1024 * 1024,
+    backup_count: int = 5,
+) -> logging.Logger:
+    """设置全局统一的日志配置"""
+    logger = logging.getLogger(name)
+
+    level_str = log_level or settings.LOG_LEVEL
+    logger.setLevel(LOG_LEVEL_MAPPING.get(level_str.upper(), logging.INFO))
+
+    if logger.handlers:
+        return logger
+
+    formatter = logging.Formatter(log_format, datefmt=date_format)
+
+    if settings.LOG_TO_CONSOLE:
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+    if settings.LOG_TO_FILE:
+        ensure_log_dir(settings.LOG_DIR)
+        log_file_path = log_file or os.path.join(settings.LOG_DIR, settings.LOG_FILE)
+        file_handler = RotatingFileHandler(
+            log_file_path,
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+            encoding="utf-8",
+        )
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    return logger
+
+
+def get_default_logger(
+    name: str | None = None,
+    log_level: str | None = None,
+    log_file: str | None = None,
+) -> logging.Logger:
+    """获取默认配置的日志器"""
+    return setup_logger(name, log_level, log_file)
+
+
+def debug(logger: logging.Logger, msg: str, *args, **kwargs) -> None:
+    """记录调试信息"""
+    logger.debug(msg, *args, **kwargs)
+
+
+def info(logger: logging.Logger, msg: str, *args, **kwargs) -> None:
+    """记录一般信息"""
+    logger.info(msg, *args, **kwargs)
+
+
+def warning(logger: logging.Logger, msg: str, *args, **kwargs) -> None:
+    """记录警告信息"""
+    logger.warning(msg, *args, **kwargs)
+
+
+def error(logger: logging.Logger, msg: str, *args, **kwargs) -> None:
+    """记录错误信息"""
+    logger.error(msg, *args, **kwargs)
+
+
+def critical(logger: logging.Logger, msg: str, *args, **kwargs) -> None:
+    """记录严重错误信息"""
+    logger.critical(msg, *args, **kwargs)
+
+
+__all__ = [
+    "setup_logger",
+    "get_default_logger",
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "critical",
+]

kit/db/__init__.py

@@ -0,0 +1,70 @@
+"""
+Copyright (c) 2021-2026 Clark Chang. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Project: sdpy
+Author: Clark Chang
+"""
+
+# date: 2026-08-23
+"""kit.db — 基于 SQLModel 的统一数据库访问层
+
+支持 PostgreSQL、MySQL、SQLite 三种数据库，提供：
+- 线程安全的引擎管理（engine.py）
+- 统一会话入口 session_scope（session.py）
+- 不提交事务的泛型 CRUDBase（crud.py）
+- @transactional 装饰器（transaction.py）
+- 统一异常体系 DBError（exceptions.py）
+- SQLModel 混入类（models/base.py）
+"""
+
+from kit.db.crud import CRUDBase
+from kit.db.engine import (
+    MYSQL,
+    POSTGRESQL,
+    SQLITE,
+    close_all_engines,
+    close_engine,
+    create_tables,
+    get_engine,
+    list_tables,
+    table_exists,
+)
+from kit.db.exceptions import DBError, NotFoundError
+from kit.db.session import get_session, session_scope
+from kit.db.transaction import transactional
+
+__all__ = [
+    # 引擎管理
+    "get_engine",
+    "create_tables",
+    "list_tables",
+    "table_exists",
+    "close_all_engines",
+    "close_engine",
+    # 常量
+    "POSTGRESQL",
+    "MYSQL",
+    "SQLITE",
+    # 会话
+    "session_scope",
+    "get_session",
+    # 异常
+    "DBError",
+    "NotFoundError",
+    # CRUD
+    "CRUDBase",
+    # 事务
+    "transactional",
+]