sycommon-python-lib 0.1.0__py3-none-any.whl

sycommon/config/Config.py

@@ -0,0 +1,73 @@
+import yaml
+
+
+class SingletonMeta(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class Config(metaclass=SingletonMeta):
+    def __init__(self, config_file='app.yaml'):
+        with open(config_file, 'r') as f:
+            self.config = yaml.safe_load(f)
+        self.MaxBytes = self.config.get('MaxBytes', 209715200)
+        self.Timeout = self.config.get('Timeout', 300000)
+        self.OCR = self.config.get('OCR', None)
+        self.INVOICE_OCR = self.config.get('INVOICE_OCR', None)
+        self.UnstructuredAPI = self.config.get('UnstructuredAPI', None)
+        self.MaxRetries = self.config.get('MaxRetries', 3)
+        self.llm_configs = []
+        self.embedding_configs = []
+        self.reranker_configs = []
+        self._process_config()
+
+    def get_llm_config(self, model_name):
+        for llm in self.llm_configs:
+            if llm.get('model') == model_name:
+                return llm
+        raise ValueError(f"No configuration found for model: {model_name}")
+
+    def get_embedding_config(self, model_name):
+        for llm in self.embedding_configs:
+            if llm.get('model') == model_name:
+                return llm
+        raise ValueError(f"No configuration found for model: {model_name}")
+
+    def get_reranker_config(self, model_name):
+        for llm in self.reranker_configs:
+            if llm.get('model') == model_name:
+                return llm
+        raise ValueError(f"No configuration found for model: {model_name}")
+
+    def _process_config(self):
+        llm_config_list = self.config.get('LLMConfig', [])
+        for llm_config in llm_config_list:
+            try:
+                # 延迟导入 LLMConfigModel
+                from sycommon.config.LLMConfig import LLMConfig
+                validated_config = LLMConfig(**llm_config)
+                self.llm_configs.append(validated_config.model_dump())
+            except ValueError as e:
+                print(f"Invalid LLM configuration: {e}")
+
+        embedding_config_list = self.config.get('EmbeddingConfig', [])
+        for embedding_config in embedding_config_list:
+            try:
+                from sycommon.config.EmbeddingConfig import EmbeddingConfig
+                validated_config = EmbeddingConfig(**embedding_config)
+                self.embedding_configs.append(validated_config.model_dump())
+            except ValueError as e:
+                print(f"Invalid LLM configuration: {e}")
+
+        reranker_config_list = self.config.get('RerankerConfig', [])
+        for reranker_config in reranker_config_list:
+            try:
+                from sycommon.config.RerankerConfig import RerankerConfig
+                validated_config = RerankerConfig(**reranker_config)
+                self.reranker_configs.append(validated_config.model_dump())
+            except ValueError as e:
+                print(f"Invalid LLM configuration: {e}")

sycommon/config/DatabaseConfig.py

@@ -0,0 +1,34 @@
+from pydantic import BaseModel
+
+
+class HikariConfig(BaseModel):
+    minimum_idle: int
+    maximum_pool_size: int
+    idle_timeout: int
+
+
+class DatabaseConfig(BaseModel):
+    url: str
+    username: str
+    password: str
+    driver_class_name: str
+    time_between_eviction_runs_millis: int
+    min_evictable_idle_time_millis: int
+    validation_query: str
+    test_while_idle: bool
+    test_on_borrow: bool
+    test_on_return: bool
+    hikari: HikariConfig
+
+
+def convert_key_to_snake_case(key):
+    import re
+    # 处理驼峰命名转换为下划线命名
+    key = re.sub(r'(?<!^)(?=[A-Z])', '_', key).lower()
+    return key.replace('-', '_')
+
+
+def convert_dict_keys(d):
+    if isinstance(d, dict):
+        return {convert_key_to_snake_case(k): convert_dict_keys(v) for k, v in d.items()}
+    return d

sycommon/config/EmbeddingConfig.py

@@ -0,0 +1,16 @@
+
+from pydantic import BaseModel
+
+
+class EmbeddingConfig(BaseModel):
+    model: str
+    provider: str
+    baseUrl: str
+    maxTokens: int
+    dimension: int
+
+    @classmethod
+    def from_config(cls, model_name: str):
+        from sycommon.config.Config import Config
+        llm_config = Config().get_embedding_config(model_name)
+        return cls(**llm_config)

sycommon/config/LLMConfig.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+
+class LLMConfig(BaseModel):
+    model: str
+    provider: str
+    baseUrl: str
+    maxTokens: int
+    vision: bool
+    callFunction: bool
+
+    @classmethod
+    def from_config(cls, model_name: str):
+        from sycommon.config.Config import Config
+        llm_config = Config().get_llm_config(model_name)
+        return cls(**llm_config)

sycommon/config/RerankerConfig.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class RerankerConfig(BaseModel):
+    model: str
+    provider: str
+    baseUrl: str
+
+    @classmethod
+    def from_config(cls, model_name: str):
+        from sycommon.config.Config import Config
+        llm_config = Config().get_reranker_config(model_name)
+        return cls(**llm_config)

sycommon/database/database_service.py

@@ -0,0 +1,79 @@
+from sqlalchemy import create_engine, text
+
+from sycommon.config.Config import SingletonMeta
+from sycommon.config.DatabaseConfig import DatabaseConfig, convert_dict_keys
+from sycommon.logging.kafka_log import SYLogger
+from sycommon.synacos.nacos_service import NacosService
+
+
+class DatabaseService(metaclass=SingletonMeta):
+    _engine = None
+
+    @staticmethod
+    def setup_database(config: dict, shareConfigKey: str):
+        common = NacosService(config).share_configs.get(shareConfigKey, {})
+        if common and common.get('spring', {}).get('datasource', None):
+            databaseConfig = common.get('spring', {}).get('datasource', None)
+            converted_dict = convert_dict_keys(databaseConfig)
+            db_config = DatabaseConfig.model_validate(converted_dict)
+            DatabaseService._engine = DatabaseConnector(db_config).engine
+
+    @staticmethod
+    def engine():
+        return DatabaseService._engine
+
+
+class DatabaseConnector(metaclass=SingletonMeta):
+    def __init__(self, db_config: DatabaseConfig):
+        # 从 DatabaseConfig 中提取数据库连接信息
+        self.db_user = db_config.username
+        self.db_password = db_config.password
+        # 提取 URL 中的主机、端口和数据库名
+        url_parts = db_config.url.split('//')[1].split('/')
+        host_port = url_parts[0].split(':')
+        self.db_host = host_port[0]
+        self.db_port = host_port[1]
+        self.db_name = url_parts[1].split('?')[0]
+
+        # 提取 URL 中的参数
+        params_str = url_parts[1].split('?')[1] if len(
+            url_parts[1].split('?')) > 1 else ''
+        params = {}
+        for param in params_str.split('&'):
+            if param:
+                key, value = param.split('=')
+                params[key] = value
+
+        # 在params中去掉指定的参数
+        for key in ['useUnicode', 'characterEncoding', 'serverTimezone', 'zeroDateTimeBehavior']:
+            if key in params:
+                del params[key]
+
+        # 构建数据库连接 URL
+        self.db_url = f'mysql+mysqlconnector://{self.db_user}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}'
+
+        SYLogger.info(f"Database URL: {self.db_url}")
+
+        # 优化连接池配置
+        self.engine = create_engine(
+            self.db_url,
+            connect_args=params,
+            pool_size=10,  # 连接池大小
+            max_overflow=20,  # 最大溢出连接数
+            pool_timeout=30,  # 连接超时时间（秒）
+            pool_recycle=3600,  # 连接回收时间（秒）
+            pool_pre_ping=True  # 每次获取连接前检查连接是否有效
+        )
+
+        # 测试
+        if not self.test_connection():
+            raise Exception("Database connection test failed")
+
+    def test_connection(self):
+        try:
+            with self.engine.connect() as connection:
+                connection.execute(text("SELECT 1"))
+                return True
+        except Exception as e:
+            SYLogger.error(f"Database connection test failed: {e}")
+            return False

sycommon/health/health_check.py

@@ -0,0 +1,17 @@
+from fastapi import FastAPI, APIRouter
+
+
+def setup_health_handler(app: FastAPI):
+    health_router = APIRouter()
+
+    @health_router.get("/actuator/health")
+    async def health_check():
+        """返回应用的健康状态"""
+        return {
+            "status": "UP",
+            "groups": ["liveness", "readiness"]
+        }
+
+    app.include_router(health_router)
+
+    return app

sycommon/health/ping.py

@@ -0,0 +1,13 @@
+from fastapi import FastAPI, APIRouter
+
+
+def setup_ping_handler(app: FastAPI):
+    ping_router = APIRouter()
+
+    @ping_router.get("/ping")
+    async def ping():
+        return {"status": "alive"}
+
+    app.include_router(ping_router)
+
+    return app