infomankit 0.3.23__py3-none-any.whl

infoman/service/utils/resolver/resp.py

@@ -0,0 +1,102 @@
+# !/usr/bin/env python
+# -*-coding:utf-8 -*-
+
+"""
+# Time       ：2025/6/25 21:51
+# Author     ：Maxwell
+# Description：
+"""
+
+import json
+import traceback
+from typing import Any, Optional, Type, Union
+from pydantic import BaseModel, ValidationError
+from infoman.service.utils.resolver.base import BaseResp
+from infoman.logger import logger
+
+
+def get_data_from_text(text) -> (bool, Any):
+    try:
+        result = json.loads(text)
+        result_code = result.get("code")
+        if result_code != 200:
+            return False, result.get("message")
+        return True, result.get("data")
+    except Exception as e:
+        logger.info(f"get_data_from_text error: {traceback.format_exc()}")
+        return False, None
+
+
+def get_data_model_from_text(
+    text: str, model: Optional[Type[BaseModel]] = None
+) -> (bool, Any):
+    """
+    从文本中解析JSON数据，并可选地使用模型验证数据。
+
+    参数:
+        text (str): 要解析的JSON文本
+        model (Optional[Type[BaseModel]]): 可选的Pydantic模型，用于验证数据
+
+    返回:
+        tuple: (bool, Any) 第一个元素表示是否成功，第二个元素是数据或错误信息
+    """
+    try:
+        result = json.loads(text)
+
+        result_code = result.get("code")
+        if result_code != "000000":
+            return False, result.get("message")
+
+        data = result.get("data")
+        if model is not None:
+            if isinstance(data, dict):
+                try:
+                    validated_data = model.parse_obj(data)
+                    return True, validated_data
+                except ValidationError as e:
+                    logger.info(f"Data validation error: {e}")
+                    return False, f"Data validation failed: {e}"
+            elif isinstance(data, list):
+                data_list = []
+                for one in data:
+                    try:
+                        validated_data = model.parse_obj(one)
+                        data_list.append(validated_data)
+                    except ValidationError as e:
+                        logger.info(f"Data validation error: {e}")
+                return True, data_list
+        return True, data
+
+    except Exception as e:
+        logger.info(f"get_data_from_text error: {traceback.format_exc()}")
+        return False, None
+
+
+def get_model_from_text(
+    text: str, model: Optional[Type[BaseModel]] = None
+) -> (bool, Any):
+    try:
+        result = json.loads(text)
+        result_code = result.get("code")
+        if result_code != "000000":
+            return False, result.get("message")
+
+        data = result.get("data")
+        if isinstance(data, list):
+            try:
+                validated_data = model.parse_obj(result)
+                return True, validated_data
+            except ValidationError as e:
+                logger.info(f"Data validation error: {e}")
+
+        if model is not None:
+            try:
+                validated_data = model.parse_obj(data)
+                return True, validated_data
+            except ValidationError as e:
+                logger.info(f"Data validation error: {e}")
+        return False, "data is null"
+
+    except Exception as e:
+        logger.info(f"get_data_from_text error: {traceback.format_exc()}")
+        return False, None

infoman/service/vector/__init__.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Vector store integration entry.
+
+Exposes a light-weight abstraction and a Qdrant implementation.
+"""
+
+from .base import VectorClient, VectorPoint, VectorFilter
+from .qdrant import QdrantVectorClient
+from .service import VectorService, get_vector_service
+
+__all__ = [
+    "VectorClient",
+    "VectorPoint",
+    "VectorFilter",
+    "QdrantVectorClient",
+    "VectorService",
+    "get_vector_service",
+]

infoman/service/vector/base.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Iterable, Mapping, Protocol, Sequence
+
+
+@dataclass
+class VectorPoint:
+    id: str | int
+    vector: Sequence[float]
+    payload: Mapping[str, Any] | None = None
+
+
+@dataclass
+class VectorFilter:
+    # Generic equality filter; concrete impl can translate
+    must: dict[str, Any] | None = None
+    must_not: dict[str, Any] | None = None
+
+
+class VectorClient(Protocol):
+    async def ensure_collection(
+        self,
+        name: str,
+        vector_size: int,
+        distance: str = "cosine",
+        on_disk: bool = False,
+    ) -> None: ...
+
+    async def upsert(
+        self,
+        collection: str,
+        points: Iterable[VectorPoint],
+        wait: bool = False,
+    ) -> None: ...
+
+    async def delete(
+        self,
+        collection: str,
+        ids: Iterable[str | int] | None = None,
+        filt: VectorFilter | None = None,
+        wait: bool = False,
+    ) -> None: ...
+
+    async def search(
+        self,
+        collection: str,
+        query_vector: Sequence[float],
+        limit: int = 10,
+        filt: VectorFilter | None = None,
+        with_vectors: bool = False,
+    ) -> list[dict[str, Any]]: ...
+

infoman/service/vector/qdrant.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Sequence
+
+from .base import VectorClient, VectorPoint, VectorFilter
+
+try:
+    from qdrant_client import AsyncQdrantClient
+    from qdrant_client.models import (
+        Distance,
+        VectorParams,
+        PointStruct,
+        Filter as QFilter,
+        FieldCondition,
+        MatchAny,
+        MatchValue,
+    )
+except Exception:  # pragma: no cover - optional dependency at import time
+    AsyncQdrantClient = None  # type: ignore
+    Distance = VectorParams = PointStruct = object  # type: ignore
+    QFilter = FieldCondition = MatchAny = MatchValue = object  # type: ignore
+
+
+_DIST_MAP = {
+    "cosine": Distance.COSINE if hasattr(Distance, "COSINE") else "cosine",
+    "dot": Distance.DOT if hasattr(Distance, "DOT") else "dot",
+    "euclid": Distance.EUCLID if hasattr(Distance, "EUCLID") else "euclid",
+}
+
+
+def _to_qfilter(f: VectorFilter | None) -> QFilter | None:
+    if f is None:
+        return None
+    must = []
+    must_not = []
+    if f.must:
+        for k, v in f.must.items():
+            if isinstance(v, (list, tuple, set)):
+                must.append(FieldCondition(key=k, match=MatchAny(any=list(v))))
+            else:
+                must.append(FieldCondition(key=k, match=MatchValue(value=v)))
+    if f.must_not:
+        for k, v in f.must_not.items():
+            if isinstance(v, (list, tuple, set)):
+                must_not.append(FieldCondition(key=k, match=MatchAny(any=list(v))))
+            else:
+                must_not.append(FieldCondition(key=k, match=MatchValue(value=v)))
+    return QFilter(must=must or None, must_not=must_not or None)
+
+
+class QdrantVectorClient(VectorClient):
+    def __init__(self, client: Any):
+        if AsyncQdrantClient is not None and not isinstance(client, AsyncQdrantClient):
+            # Allow duck typing (useful in tests), but warn in docs
+            pass
+        self.client = client
+
+    async def ensure_collection(
+        self, name: str, vector_size: int, distance: str = "cosine", on_disk: bool = False
+    ) -> None:
+        dist = _DIST_MAP.get(distance, _DIST_MAP["cosine"])
+        exists = await self.client.collection_exists(collection_name=name)
+        if exists:
+            return
+        await self.client.create_collection(
+            collection_name=name,
+            vectors_config=VectorParams(size=vector_size, distance=dist, on_disk=on_disk),
+        )
+
+    async def upsert(self, collection: str, points: Iterable[VectorPoint], wait: bool = False) -> None:
+        payloads = []
+        qpoints: list[PointStruct] = []
+        for p in points:
+            payload = dict(p.payload) if p.payload else None
+            payloads.append(payload)
+            qpoints.append(PointStruct(id=p.id, vector=list(p.vector), payload=payload))
+        if not qpoints:
+            return
+        await self.client.upsert(collection_name=collection, points=qpoints, wait=wait)
+
+    async def delete(
+        self,
+        collection: str,
+        ids: Iterable[str | int] | None = None,
+        filt: VectorFilter | None = None,
+        wait: bool = False,
+    ) -> None:
+        if ids:
+            await self.client.delete(collection_name=collection, points=list(ids), wait=wait)
+            return
+        qf = _to_qfilter(filt)
+        if qf is not None:
+            await self.client.delete(collection_name=collection, filter=qf, wait=wait)
+
+    async def search(
+        self,
+        collection: str,
+        query_vector: Sequence[float],
+        limit: int = 10,
+        filt: VectorFilter | None = None,
+        with_vectors: bool = False,
+    ) -> list[dict[str, Any]]:
+        qf = _to_qfilter(filt)
+        res = await self.client.search(
+            collection_name=collection,
+            query_vector=list(query_vector),
+            limit=limit,
+            filter=qf,
+            with_vectors=with_vectors,
+        )
+        results: list[dict[str, Any]] = []
+        for r in res:
+            results.append(
+                {
+                    "id": r.id,
+                    "score": r.score,
+                    "payload": getattr(r, "payload", None),
+                    "vector": getattr(r, "vector", None) if with_vectors else None,
+                }
+            )
+        return results
+

infoman/service/vector/service.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from fastapi import FastAPI
+
+from .base import VectorPoint, VectorFilter
+from . import VectorClient
+from infoman.service.models.type.embed import (
+    EmbedModel,
+    EmbedCollection,
+    EmbedDataType,
+)
+
+
+class VectorService:
+    """High-level vector operations bound to Embed* config.
+
+    This provides convenience methods using `EmbedModel` and `EmbedDataType`
+    to compute collection names and dimensions.
+    """
+
+    def __init__(self, vector_client: VectorClient):
+        self.vc = vector_client
+
+    @staticmethod
+    def collection_name(data_type: EmbedDataType, model: EmbedModel) -> str:
+        return EmbedCollection.collection_name(data_type, model)
+
+    async def ensure_collection(self, data_type: EmbedDataType, model: EmbedModel) -> str:
+        name = self.collection_name(data_type, model)
+        await self.vc.ensure_collection(name, model.value.dem, distance="cosine")
+        return name
+
+    async def upsert(
+        self,
+        data_type: EmbedDataType,
+        model: EmbedModel,
+        points: list[VectorPoint],
+        wait: bool = False,
+    ) -> str:
+        name = self.collection_name(data_type, model)
+        await self.vc.upsert(name, points, wait=wait)
+        return name
+
+    async def search(
+        self,
+        data_type: EmbedDataType,
+        model: EmbedModel,
+        query_vector: Sequence[float],
+        limit: int = 10,
+        filt: VectorFilter | None = None,
+        with_vectors: bool = False,
+    ) -> list[dict[str, Any]]:
+        name = self.collection_name(data_type, model)
+        return await self.vc.search(name, query_vector, limit, filt, with_vectors)
+
+
+def get_vector_service(app: FastAPI) -> VectorService | None:
+    vc = getattr(app.state, "vector_client", None)
+    if vc is None:
+        return None
+    return VectorService(vc)
+

infoman/utils/__init__.py

@@ -0,0 +1,2 @@
+# !/usr/bin/env python
+# -*-coding:utf-8 -*-

infoman/utils/decorators/__init__.py

@@ -0,0 +1,8 @@
+# !/usr/bin/env python
+# -*-coding:utf-8 -*-
+
+"""
+# Time       ：2025/6/22 12:06
+# Author     ：Maxwell
+# Description：
+"""

infoman/utils/decorators/cache.py

@@ -0,0 +1,137 @@
+# !/usr/bin/env python
+# -*-coding:utf-8 -*-
+
+"""
+# Time       ：2025/6/22 12:11
+# Author     ：Maxwell
+# Description：
+"""
+import time
+import functools
+from typing import Callable, TypeVar, Any, Optional, Union, Dict, List, Tuple
+from infoman.logger import logger
+from inspect import iscoroutinefunction
+
+T = TypeVar("T")
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def cache(
+    ttl: Optional[int] = 60,  # 缓存生存时间(秒)
+    max_size: int = 128,  # 最大缓存条目数
+    key_func: Optional[Callable] = None,  # 自定义缓存键生成函数
+) -> Callable[[F], F]:
+    """
+    缓存装饰器 - 缓存函数结果以提高性能
+
+    参数:
+        ttl: 缓存生存时间(秒)，None表示永不过期
+        max_size: 最大缓存条目数
+        key_func: 自定义缓存键生成函数
+
+    示例:
+        @cache(ttl=300)  # 缓存5分钟
+        def get_user_data(user_id):
+            # 获取用户数据的代码
+    """
+    cache_dict: Dict[str, Tuple[Any, Optional[float]]] = {}
+
+    def make_key(*args, **kwargs) -> str:
+        """生成缓存键"""
+        if key_func:
+            return str(key_func(*args, **kwargs))
+
+        # 默认键生成逻辑
+        key_parts = [str(arg) for arg in args]
+        key_parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
+        return ":".join(key_parts)
+
+    def is_expired(timestamp: Optional[float]) -> bool:
+        """检查缓存是否过期"""
+        if timestamp is None:  # 永不过期
+            return False
+        return time.time() > timestamp
+
+    def cleanup_cache() -> None:
+        """清理过期缓存"""
+        if ttl is None:
+            return
+
+        expired_keys = [
+            k
+            for k, (_, exp_time) in cache_dict.items()
+            if exp_time is not None and is_expired(exp_time)
+        ]
+
+        for k in expired_keys:
+            del cache_dict[k]
+
+    def decorator(func: F) -> F:
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            # 清理过期缓存
+            cleanup_cache()
+
+            # 生成缓存键
+            key = make_key(*args, **kwargs)
+
+            # 检查缓存
+            if key in cache_dict:
+                value, exp_time = cache_dict[key]
+                if not is_expired(exp_time):
+                    return value
+
+            # 缓存未命中，执行函数
+            result = await func(*args, **kwargs)
+
+            # 更新缓存
+            if len(cache_dict) >= max_size:
+                # 简单的LRU策略：删除第一个条目
+                if cache_dict:
+                    cache_dict.pop(next(iter(cache_dict)))
+
+            expiration = (time.time() + ttl) if ttl is not None else None
+            cache_dict[key] = (result, expiration)
+
+            return result
+
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            # 清理过期缓存
+            cleanup_cache()
+
+            # 生成缓存键
+            key = make_key(*args, **kwargs)
+
+            # 检查缓存
+            if key in cache_dict:
+                value, exp_time = cache_dict[key]
+                if not is_expired(exp_time):
+                    return value
+
+            # 缓存未命中，执行函数
+            result = func(*args, **kwargs)
+
+            # 更新缓存
+            if len(cache_dict) >= max_size:
+                # 简单的LRU策略：删除第一个条目
+                if cache_dict:
+                    cache_dict.pop(next(iter(cache_dict)))
+
+            expiration = (time.time() + ttl) if ttl is not None else None
+            cache_dict[key] = (result, expiration)
+
+            return result
+
+        # 添加清除缓存的辅助方法
+        def clear_cache():
+            cache_dict.clear()
+
+        if iscoroutinefunction(func):
+            async_wrapper.clear_cache = clear_cache
+            return async_wrapper
+        else:
+            sync_wrapper.clear_cache = clear_cache
+            return sync_wrapper
+
+    return decorator

infoman/utils/decorators/retry.py

@@ -0,0 +1,99 @@
+# !/usr/bin/env python
+# -*-coding:utf-8 -*-
+
+"""
+# Time       ：2025/6/22 12:10
+# Author     ：Maxwell
+# Description：
+"""
+import time
+import asyncio
+import functools
+from typing import Callable, TypeVar, Any, Optional, Union, Dict, List, Tuple
+from infoman.logger import logger
+from inspect import iscoroutinefunction
+
+T = TypeVar("T")
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def retry(
+    max_attempts: int = 3,
+    delay_seconds: float = 1.0,
+    backoff_factor: float = 2.0,
+    exceptions: Tuple[Exception, ...] = (Exception,),
+    logger_name: Optional[str] = None,
+) -> Callable[[F], F]:
+    """
+    重试装饰器 - 在发生特定异常时自动重试函数
+
+    参数:
+        max_attempts: 最大尝试次数
+        delay_seconds: 初始延迟时间(秒)
+        backoff_factor: 退避因子(每次重试后延迟时间的乘数)
+        exceptions: 触发重试的异常类型
+        logger_name: 自定义日志记录器名称
+
+    示例:
+        @retry(max_attempts=5, exceptions=(ConnectionError, TimeoutError))
+        async def connect_to_service():
+            # 连接服务的代码
+    """
+
+    def decorator(func: F) -> F:
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            attempt = 1
+            current_delay = delay_seconds
+
+            while True:
+                try:
+                    return await func(*args, **kwargs)
+                except exceptions as e:
+                    if attempt >= max_attempts:
+                        logger.error(
+                            f"Function {func.__name__} failed after {max_attempts} attempts. "
+                            f"Last error: {str(e)}"
+                        )
+                        raise
+
+                    logger.warning(
+                        f"Attempt {attempt}/{max_attempts} for {func.__name__} failed: {str(e)}. "
+                        f"Retrying in {current_delay:.2f}s"
+                    )
+
+                    await asyncio.sleep(current_delay)
+                    attempt += 1
+                    current_delay *= backoff_factor
+
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            attempt = 1
+            current_delay = delay_seconds
+
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    if attempt >= max_attempts:
+                        logger.error(
+                            f"Function {func.__name__} failed after {max_attempts} attempts. "
+                            f"Last error: {str(e)}"
+                        )
+                        raise
+
+                    logger.warning(
+                        f"Attempt {attempt}/{max_attempts} for {func.__name__} failed: {str(e)}. "
+                        f"Retrying in {current_delay:.2f}s"
+                    )
+
+                    time.sleep(current_delay)
+                    attempt += 1
+                    current_delay *= backoff_factor
+
+        if iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+
+    return decorator