sycommon-python-lib 0.1.46__py3-none-any.whl → 0.1.57b1__py3-none-any.whl

sycommon/config/Config.py

@@ -15,14 +15,13 @@ class Config(metaclass=SingletonMeta):
         with open(config_file, 'r', encoding='utf-8') as f:
             self.config = yaml.safe_load(f)
         self.MaxBytes = self.config.get('MaxBytes', 209715200)
-        self.Timeout = self.config.get('Timeout', 300000)
-        self.OCR = self.config.get('OCR', None)
-        self.INVOICE_OCR = self.config.get('INVOICE_OCR', None)
-        self.UnstructuredAPI = self.config.get('UnstructuredAPI', None)
+        self.Timeout = self.config.get('Timeout', 600000)
         self.MaxRetries = self.config.get('MaxRetries', 3)
         self.llm_configs = []
         self.embedding_configs = []
         self.reranker_configs = []
+        self.sentry_configs = []
+        self.langfuse_configs = []
         self._process_config()
 
     def get_llm_config(self, model_name):
@@ -43,6 +42,18 @@ class Config(metaclass=SingletonMeta):
                 return llm
         raise ValueError(f"No configuration found for model: {model_name}")
 
+    def get_sentry_config(self, name):
+        for sentry in self.sentry_configs:
+            if sentry.get('name') == name:
+                return sentry
+        raise ValueError(f"No configuration found for server: {name}")
+
+    def get_langfuse_config(self, name):
+        for langfuse in self.langfuse_configs:
+            if langfuse.get('name') == name:
+                return langfuse
+        raise ValueError(f"No configuration found for server: {name}")
+
     def _process_config(self):
         llm_config_list = self.config.get('LLMConfig', [])
         for llm_config in llm_config_list:
@@ -71,3 +82,17 @@ class Config(metaclass=SingletonMeta):
                 self.reranker_configs.append(validated_config.model_dump())
             except ValueError as e:
                 print(f"Invalid LLM configuration: {e}")
+
+        sentry_config_list = self.config.get('SentryConfig', [])
+        for sentry_config in sentry_config_list:
+            try:
+                from sycommon.config.SentryConfig import SentryConfig
+                validated_config = SentryConfig(**sentry_config)
+                self.sentry_configs.append(validated_config.model_dump())
+            except ValueError as e:
+                print(f"Invalid Sentry configuration: {e}")
+
+    def set_attr(self, share_configs: dict):
+        self.config = {**self.config, **
+                       share_configs.get('llm', {}), **share_configs}
+        self._process_config()

sycommon/config/LangfuseConfig.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+
+class LangfuseConfig(BaseModel):
+    name: str
+    secretKey: str
+    publicKey: str
+    baseUrl: str
+    enable: bool
+
+    @classmethod
+    def from_config(cls, server_name: str):
+        from sycommon.config.Config import Config
+        langfuse_config = Config().get_langfuse_config(server_name)
+        return cls(**langfuse_config)

sycommon/config/RerankerConfig.py

@@ -5,6 +5,7 @@ class RerankerConfig(BaseModel):
     model: str
     provider: str
     baseUrl: str
+    maxTokens: int
 
     @classmethod
     def from_config(cls, model_name: str):

sycommon/config/SentryConfig.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class SentryConfig(BaseModel):
+    name: str
+    dsn: str
+    enable: bool
+
+    @classmethod
+    def from_config(cls, server_name: str):
+        from sycommon.config.Config import Config
+        sentry_config = Config().get_sentry_config(server_name)
+        return cls(**sentry_config)

sycommon/database/async_base_db_service.py

@@ -0,0 +1,36 @@
+from contextlib import asynccontextmanager
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.async_database_service import AsyncDatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+
+class AsyncBaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装异步会话管理功能"""
+
+    def __init__(self):
+        # 获取异步引擎 (假设 DatabaseService.engine() 返回的是 AsyncEngine)
+        self.engine = AsyncDatabaseService.engine()
+
+        # 创建异步 Session 工厂
+        # class_=AsyncSession 是必须的，用于指定生成的是异步会话
+        self.Session = async_sessionmaker(
+            bind=self.engine,
+            class_=AsyncSession,
+            expire_on_commit=False
+        )
+
+    @asynccontextmanager
+    async def session(self):
+        """
+        异步数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        async with self.Session() as session:
+            try:
+                yield session
+                await session.commit()
+            except Exception as e:
+                await session.rollback()
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise

sycommon/database/async_database_service.py

@@ -0,0 +1,96 @@
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+from sqlalchemy import text
+
+from sycommon.config.Config import SingletonMeta
+from sycommon.config.DatabaseConfig import DatabaseConfig, convert_dict_keys
+from sycommon.logging.kafka_log import SYLogger
+from sycommon.logging.async_sql_logger import AsyncSQLTraceLogger
+from sycommon.synacos.nacos_service import NacosService
+
+
+class AsyncDatabaseService(metaclass=SingletonMeta):
+    _engine = None
+
+    @staticmethod
+    async def setup_database(config: dict, shareConfigKey: str):
+        common = NacosService(config).share_configs.get(shareConfigKey, {})
+        if common and common.get('spring', {}).get('datasource', None):
+            databaseConfig = common.get('spring', {}).get('datasource', None)
+            converted_dict = convert_dict_keys(databaseConfig)
+            db_config = DatabaseConfig.model_validate(converted_dict)
+
+            # 初始化 DatabaseConnector (传入配置)
+            connector = AsyncDatabaseConnector(db_config)
+
+            # 赋值 engine
+            AsyncDatabaseService._engine = connector.engine
+
+            # 执行异步测试连接
+            if not await connector.test_connection():
+                raise Exception("Database connection test failed")
+
+    @staticmethod
+    def engine():
+        return AsyncDatabaseService._engine
+
+
+class AsyncDatabaseConnector(metaclass=SingletonMeta):
+    def __init__(self, db_config: DatabaseConfig):
+        # 从 DatabaseConfig 中提取数据库连接信息
+        self.db_user = db_config.username
+        self.db_password = db_config.password
+
+        # 提取 URL 中的主机、端口和数据库名
+        url_parts = db_config.url.split('//')[1].split('/')
+        host_port = url_parts[0].split(':')
+        self.db_host = host_port[0]
+        self.db_port = host_port[1]
+        self.db_name = url_parts[1].split('?')[0]
+
+        # 提取 URL 中的参数
+        params_str = url_parts[1].split('?')[1] if len(
+            url_parts[1].split('?')) > 1 else ''
+        params = {}
+        for param in params_str.split('&'):
+            if param:
+                key, value = param.split('=')
+                params[key] = value
+
+        # 在params中去掉指定的参数
+        for key in ['useUnicode', 'characterEncoding', 'serverTimezone', 'zeroDateTimeBehavior']:
+            if key in params:
+                del params[key]
+
+        # 构建数据库连接 URL
+        # 注意：这里将 mysqlconnector 替换为 aiomysql 以支持异步
+        self.db_url = f'mysql+aiomysql://{self.db_user}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}'
+
+        SYLogger.info(f"Database URL: {self.db_url}")
+
+        # 优化连接池配置
+        # 使用 create_async_engine 替代 create_engine
+        self.engine = create_async_engine(
+            self.db_url,
+            connect_args=params,
+            pool_size=10,  # 连接池大小
+            max_overflow=20,  # 最大溢出连接数
+            pool_timeout=30,  # 连接超时时间（秒）
+            pool_recycle=3600,  # 连接回收时间（秒）
+            pool_pre_ping=True,  # 每次获取连接前检查连接是否有效
+            echo=False,  # 打印 SQL 语句
+        )
+
+        # 注册 SQL 日志拦截器 (注意：SQLTraceLogger 需要支持异步引擎，或者您可能需要调整日志逻辑)
+        # 假设 SQLTraceLogger.setup_sql_logging 能够处理 AsyncEngine
+        AsyncSQLTraceLogger.setup_sql_logging(self.engine)
+
+    async def test_connection(self):
+        try:
+            # 异步上下文管理器
+            async with self.engine.connect() as connection:
+                # 执行简单查询
+                await connection.execute(text("SELECT 1"))
+                return True
+        except Exception as e:
+            SYLogger.error(f"Database connection test failed: {e}")
+            return False

sycommon/llm/embedding.py

@@ -0,0 +1,204 @@
+import asyncio
+import json
+import aiohttp
+from typing import Union, List, Optional
+
+from sycommon.config.Config import SingletonMeta
+from sycommon.config.EmbeddingConfig import EmbeddingConfig
+from sycommon.config.RerankerConfig import RerankerConfig
+from sycommon.logging.kafka_log import SYLogger
+
+
+class Embedding(metaclass=SingletonMeta):
+    def __init__(self):
+        # 1. 并发限制
+        self.max_concurrency = 20
+        # 保留默认模型名称
+        self.default_embedding_model = "bge-large-zh-v1.5"
+        self.default_reranker_model = "bge-reranker-large"
+
+        # 初始化默认模型的基础URL
+        self.embeddings_base_url = EmbeddingConfig.from_config(
+            self.default_embedding_model).baseUrl
+        self.reranker_base_url = RerankerConfig.from_config(
+            self.default_reranker_model).baseUrl
+
+        # 并发信号量
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
+        # 全局默认超时：永不超时（None）
+        self.default_timeout = aiohttp.ClientTimeout(total=None)
+
+    async def _get_embeddings_http_async(
+        self,
+        input: Union[str, List[str]],
+        encoding_format: str = None,
+        model: str = None,
+        timeout: aiohttp.ClientTimeout = None,
+        **kwargs
+    ):
+        async with self.semaphore:
+            # 优先使用传入的超时，无则用全局默认
+            request_timeout = timeout or self.default_timeout
+
+            # 优先使用传入的模型名，无则用默认值
+            target_model = model or self.default_embedding_model
+            target_base_url = EmbeddingConfig.from_config(target_model).baseUrl
+            url = f"{target_base_url}/v1/embeddings"
+
+            request_body = {
+                "model": target_model,
+                "input": input,
+                "encoding_format": encoding_format or "float"
+            }
+            request_body.update(kwargs)
+
+            try:
+                async with aiohttp.ClientSession(timeout=request_timeout) as session:
+                    async with session.post(url, json=request_body) as response:
+                        if response.status != 200:
+                            error_detail = await response.text()
+                            SYLogger.error(
+                                f"Embedding request failed (model: {target_model}): {error_detail}")
+                            return None
+                        return await response.json()
+            except asyncio.TimeoutError:
+                SYLogger.error(
+                    f"Embedding request timeout (model: {target_model})")
+                return None
+            except Exception as e:
+                SYLogger.error(
+                    f"Embedding request unexpected error (model: {target_model}): {str(e)}")
+                return None
+
+    async def _get_reranker_http_async(
+        self,
+        documents: List[str],
+        query: str,
+        top_n: Optional[int] = None,
+        model: str = None,
+        max_chunks_per_doc: Optional[int] = None,
+        return_documents: Optional[bool] = True,
+        return_len: Optional[bool] = True,
+        timeout: aiohttp.ClientTimeout = None,
+        **kwargs
+    ):
+        async with self.semaphore:
+            # 优先使用传入的超时，无则用全局默认
+            request_timeout = timeout or self.default_timeout
+
+            # 优先使用传入的模型名，无则用默认值
+            target_model = model or self.default_reranker_model
+            target_base_url = RerankerConfig.from_config(target_model).baseUrl
+            url = f"{target_base_url}/v1/rerank"
+
+            request_body = {
+                "model": target_model,
+                "documents": documents,
+                "query": query,
+                "top_n": top_n or len(documents),
+                "max_chunks_per_doc": max_chunks_per_doc,
+                "return_documents": return_documents,
+                "return_len": return_len,
+                "kwargs": json.dumps(kwargs),
+            }
+            request_body.update(kwargs)
+
+            try:
+                async with aiohttp.ClientSession(timeout=request_timeout) as session:
+                    async with session.post(url, json=request_body) as response:
+                        if response.status != 200:
+                            error_detail = await response.text()
+                            SYLogger.error(
+                                f"Rerank request failed (model: {target_model}): {error_detail}")
+                            return None
+                        return await response.json()
+            except asyncio.TimeoutError:
+                SYLogger.error(
+                    f"Rerank request timeout (model: {target_model})")
+                return None
+            except Exception as e:
+                SYLogger.error(
+                    f"Rerank request unexpected error (model: {target_model}): {str(e)}")
+                return None
+
+    async def get_embeddings(
+        self,
+        corpus: List[str],
+        model: str = None,
+        timeout: Optional[Union[int, float]] = None
+    ):
+        """
+        获取语料库的嵌入向量，结果顺序与输入语料库顺序一致
+
+        Args:
+            corpus: 待生成嵌入向量的文本列表
+            model: 可选，指定使用的embedding模型名称，默认使用bge-large-zh-v1.5
+            timeout: 可选，超时时间（秒）：
+                     - 传int/float：表示总超时时间（秒）
+                     - 不传/None：使用默认永不超时配置
+        """
+        request_timeout = None
+        if timeout is not None:
+            if isinstance(timeout, (int, float)):
+                request_timeout = aiohttp.ClientTimeout(total=timeout)
+            else:
+                SYLogger.warning(
+                    f"Invalid timeout type: {type(timeout)}, must be int/float, use default timeout")
+
+        SYLogger.info(
+            f"Requesting embeddings for corpus: {corpus} (model: {model or self.default_embedding_model}, max_concurrency: {self.max_concurrency}, timeout: {timeout or 'None'})")
+
+        # 给每个异步任务传入模型名称和超时配置
+        tasks = [self._get_embeddings_http_async(
+            text, model=model, timeout=request_timeout) for text in corpus]
+        results = await asyncio.gather(*tasks)
+
+        vectors = []
+        for result in results:
+            if result is None:
+                zero_vector = [0.0] * 1024
+                vectors.append(zero_vector)
+                SYLogger.warning(
+                    f"Embedding request failed, append zero vector (1024D)")
+                continue
+            for item in result["data"]:
+                vectors.append(item["embedding"])
+
+        SYLogger.info(
+            f"Embeddings for corpus: {corpus} created (model: {model or self.default_embedding_model})")
+        return vectors
+
+    async def get_reranker(
+        self,
+        top_results: List[str],
+        query: str,
+        model: str = None,
+        timeout: Optional[Union[int, float]] = None
+    ):
+        """
+        对搜索结果进行重排序
+
+        Args:
+            top_results: 待重排序的文本列表
+            query: 排序参考的查询语句
+            model: 可选，指定使用的reranker模型名称，默认使用bge-reranker-large
+            timeout: 可选，超时时间（秒）：
+                     - 传int/float：表示总超时时间（秒）
+                     - 不传/None：使用默认永不超时配置
+        """
+        request_timeout = None
+        if timeout is not None:
+            if isinstance(timeout, (int, float)):
+                request_timeout = aiohttp.ClientTimeout(total=timeout)
+            else:
+                SYLogger.warning(
+                    f"Invalid timeout type: {type(timeout)}, must be int/float, use default timeout")
+
+        SYLogger.info(
+            f"Requesting reranker for top_results: {top_results} (model: {model or self.default_reranker_model}, max_concurrency: {self.max_concurrency}, timeout: {timeout or 'None'})")
+
+        data = await self._get_reranker_http_async(
+            top_results, query, model=model, timeout=request_timeout)
+        SYLogger.info(
+            f"Reranker for top_results: {top_results} completed (model: {model or self.default_reranker_model})")
+        return data

sycommon/llm/get_llm.py

@@ -0,0 +1,37 @@
+from sycommon.llm.llm_logger import LLMLogger
+from langchain.chat_models import init_chat_model
+from sycommon.config.LLMConfig import LLMConfig
+from sycommon.llm.sy_langfuse import LangfuseInitializer
+from sycommon.llm.usage_token import LLMWithAutoTokenUsage
+
+
+def get_llm(
+    model: str = None,
+    streaming: bool = False
+) -> LLMWithAutoTokenUsage:
+    if not model:
+        model = "Qwen2.5-72B"
+
+    llmConfig = LLMConfig.from_config(model)
+    if not llmConfig:
+        raise Exception(f"无效的模型配置：{model}")
+
+    # 初始化Langfuse
+    langfuse_callbacks, langfuse = LangfuseInitializer.get()
+
+    callbacks = [LLMLogger()] + langfuse_callbacks
+
+    llm = init_chat_model(
+        model_provider=llmConfig.provider,
+        model=llmConfig.model,
+        base_url=llmConfig.baseUrl,
+        api_key="-",
+        temperature=0.1,
+        streaming=streaming,
+        callbacks=callbacks
+    )
+
+    if llm is None:
+        raise Exception(f"初始化原始LLM实例失败：{model}")
+
+    return LLMWithAutoTokenUsage(llm, langfuse)

sycommon/llm/llm_logger.py

@@ -0,0 +1,126 @@
+from langchain_core.callbacks import AsyncCallbackHandler
+from typing import Any, Dict, List
+from langchain_core.outputs import GenerationChunk, ChatGeneration
+from langchain_core.messages import BaseMessage
+
+from sycommon.logging.kafka_log import SYLogger
+
+
+class LLMLogger(AsyncCallbackHandler):
+    """
+    通用LLM日志回调处理器，同时支持：
+    - 同步调用（如 chain.invoke()）
+    - 异步调用（如 chain.astream()）
+    - 聊天模型调用
+    """
+
+    # ------------------------------
+    # 同步回调方法（处理 invoke 等同步调用）
+    # ------------------------------
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        model_name = serialized.get('name', 'unknown')
+        SYLogger.info(
+            f"[同步] LLM调用开始 | 模型: {model_name} | 提示词数: {len(prompts)}")
+        self._log_prompts(prompts)
+
+    def on_chat_model_start(
+        self,
+        serialized: Dict[str, Any],
+        messages: List[List[BaseMessage]],
+        **kwargs: Any
+    ) -> None:
+        model_name = serialized.get('name', 'unknown')
+        SYLogger.info(
+            f"[同步] 聊天模型调用开始 | 模型: {model_name} | 消息组数: {len(messages)}")
+        self._log_chat_messages(messages)
+
+    def on_llm_end(self, response: Any, **kwargs: Any) -> None:
+        # 处理普通LLM结果
+        if hasattr(response, 'generations') and all(
+            isinstance(gen[0], GenerationChunk) for gen in response.generations
+        ):
+            for i, generation in enumerate(response.generations):
+                result = generation[0].text
+                SYLogger.info(
+                    f"[同步] LLM调用结束 | 结果 #{i+1} 长度: {len(result)}")
+                self._log_result(result, i+1)
+        # 处理聊天模型结果
+        elif hasattr(response, 'generations') and all(
+            isinstance(gen[0], ChatGeneration) for gen in response.generations
+        ):
+            for i, generation in enumerate(response.generations):
+                result = generation[0].message.content
+                SYLogger.info(
+                    f"[同步] 聊天模型调用结束 | 结果 #{i+1} 长度: {len(result)}")
+                self._log_result(result, i+1)
+
+    def on_llm_error(self, error: Exception, **kwargs: Any) -> None:
+        if isinstance(error, GeneratorExit):
+            SYLogger.info("[同步] LLM生成器正常关闭")
+            return
+        SYLogger.error(f"[同步] LLM调用出错: {str(error)}")
+
+    # ------------------------------
+    # 异步回调方法（处理 astream 等异步调用）
+    # ------------------------------
+    async def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        model_name = serialized.get('name', 'unknown')
+        SYLogger.info(
+            f"[异步] LLM调用开始 | 模型: {model_name} | 提示词数: {len(prompts)}")
+        self._log_prompts(prompts)
+
+    async def on_chat_model_start(
+        self,
+        serialized: Dict[str, Any],
+        messages: List[List[BaseMessage]],
+        **kwargs: Any
+    ) -> None:
+        model_name = serialized.get('name', 'unknown')
+        SYLogger.info(
+            f"[异步] 聊天模型调用开始 | 模型: {model_name} | 消息组数: {len(messages)}")
+        self._log_chat_messages(messages)
+
+    async def on_llm_end(self, response: Any, **kwargs: Any) -> None:
+        # 处理普通LLM结果
+        if hasattr(response, 'generations') and all(
+            isinstance(gen[0], GenerationChunk) for gen in response.generations
+        ):
+            for i, generation in enumerate(response.generations):
+                result = generation[0].text
+                SYLogger.info(
+                    f"[异步] LLM调用结束 | 结果 #{i+1} 长度: {len(result)}")
+                self._log_result(result, i+1)
+        # 处理聊天模型结果
+        elif hasattr(response, 'generations') and all(
+            isinstance(gen[0], ChatGeneration) for gen in response.generations
+        ):
+            for i, generation in enumerate(response.generations):
+                result = generation[0].message.content
+                SYLogger.info(
+                    f"[异步] 聊天模型调用结束 | 结果 #{i+1} 长度: {len(result)}")
+                self._log_result(result, i+1)
+
+    async def on_llm_error(self, error: Exception, **kwargs: Any) -> None:
+        if isinstance(error, GeneratorExit):
+            SYLogger.info("[异步] LLM生成器正常关闭")
+            return
+        SYLogger.error(f"[异步] LLM调用出错: {str(error)}")
+
+    # ------------------------------
+    # 共享工具方法（避免代码重复）
+    # ------------------------------
+    def _log_prompts(self, prompts: List[str]) -> None:
+        """记录提示词"""
+        for i, prompt in enumerate(prompts):
+            SYLogger.info(f"提示词 #{i+1}:\n{prompt}")
+
+    def _log_chat_messages(self, messages: List[List[BaseMessage]]) -> None:
+        """记录聊天模型的消息"""
+        for i, message_group in enumerate(messages):
+            SYLogger.info(f"消息组 #{i+1}:")
+            for msg in message_group:
+                SYLogger.info(f"  {msg.type}: {msg.content}")
+
+    def _log_result(self, result: str, index: int) -> None:
+        """记录结果"""
+        SYLogger.info(f"结果 #{index}:\n{result}")