algo-backend-framework 0.0.1__tar.gz

algo_backend_framework-0.0.1/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: algo-backend-framework
+Version: 0.0.1
+Summary: Ctcdn algorithm backend framework
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.128.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: prometheus-client>=0.24.1
+Requires-Dist: psutil>=7.2.1
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: python-dotenv>=1.2.1
+Requires-Dist: uvicorn>=0.40.0
+
+# algo-backend-framework-python
+
+# 功能概述
+- 配置
+- 错误码、异常、参数检验报错转换
+- 标准响应体、x-request-id
+- 日志框架支持：loguru、nb_log
+- 多进程prometheus指标
+- otel
+
+## 分支规范
+- master分支: 归档分支
+- release/release-×.×.×: 发布分支，×.×.×为版本号；修改[pyproject.toml](pyproject.toml)中的version构建并发布包
+- dev分支: 开发分支；合并至release分支时，需要提mr，mr需要填写版本号和功能描述；需要充分自测和review之后才能合并至release
+- feat/feat-×××××: 功能分支；合并至dev时需要mr，mr需要填写功能名称和功能描述；新功能或较大重构需要建feat分支，较小的改动不必开feat分支
+
+## 本地调试
+
+本地开发需要打包验证，则先构建包（但不要发布），然后在本地目录离线安装
+```shell
+# 构建包
+uv build
+# 拷贝wheel包到本地测试目录下执行以下命令
+uv add algo_backend_framework-*.*.*-py3-none-any.whl
+# 调试时发现包有问题，只需要remove后重新打包，并重新add
+uv remove algo-backend-framework
+```
+
+## 开发约定
+- 代码提交前进行sonar扫描和语法检查，并进行代码格式化
+    ```
+    uv run ruff format algo_backend/×××××
+    ```
+- 子模块__init__.py文件需要声明包内暴露模块，不建议用户通过包内文件名导入，以减少后期微调重构引起的不兼容
+- 跨模块导入一律使用`from algo_backend.× import 模块名`的全限定包名，同级目录下导入从使用`from . import `
+
+
+## 发布
+- 在release分支进行构建和推送
+```shell
+uv build
+# 推送ctcdn nexus
+uv publish --index ctcdn-pypi-upload
+# 推送pypi
+uv publish
+```

algo_backend_framework-0.0.1/README.md

@@ -0,0 +1,46 @@
+# algo-backend-framework-python
+
+# 功能概述
+- 配置
+- 错误码、异常、参数检验报错转换
+- 标准响应体、x-request-id
+- 日志框架支持：loguru、nb_log
+- 多进程prometheus指标
+- otel
+
+## 分支规范
+- master分支: 归档分支
+- release/release-×.×.×: 发布分支，×.×.×为版本号；修改[pyproject.toml](pyproject.toml)中的version构建并发布包
+- dev分支: 开发分支；合并至release分支时，需要提mr，mr需要填写版本号和功能描述；需要充分自测和review之后才能合并至release
+- feat/feat-×××××: 功能分支；合并至dev时需要mr，mr需要填写功能名称和功能描述；新功能或较大重构需要建feat分支，较小的改动不必开feat分支
+
+## 本地调试
+
+本地开发需要打包验证，则先构建包（但不要发布），然后在本地目录离线安装
+```shell
+# 构建包
+uv build
+# 拷贝wheel包到本地测试目录下执行以下命令
+uv add algo_backend_framework-*.*.*-py3-none-any.whl
+# 调试时发现包有问题，只需要remove后重新打包，并重新add
+uv remove algo-backend-framework
+```
+
+## 开发约定
+- 代码提交前进行sonar扫描和语法检查，并进行代码格式化
+    ```
+    uv run ruff format algo_backend/×××××
+    ```
+- 子模块__init__.py文件需要声明包内暴露模块，不建议用户通过包内文件名导入，以减少后期微调重构引起的不兼容
+- 跨模块导入一律使用`from algo_backend.× import 模块名`的全限定包名，同级目录下导入从使用`from . import `
+
+
+## 发布
+- 在release分支进行构建和推送
+```shell
+uv build
+# 推送ctcdn nexus
+uv publish --index ctcdn-pypi-upload
+# 推送pypi
+uv publish
+```

algo_backend_framework-0.0.1/algo_backend/config/__init__.py

@@ -0,0 +1,8 @@
+from dotenv import load_dotenv
+
+load_dotenv(".env")
+
+from .basic_config import ErrorCodeConfig, ServiceConfig
+from .loguru_config import LoguruConfig
+
+__all__ = ["LoguruConfig", "ServiceConfig", "ErrorCodeConfig"]

algo_backend_framework-0.0.1/algo_backend/config/basic_config.py

@@ -0,0 +1,13 @@
+import os
+
+from algo_backend.utils import OsAttrMeta
+
+
+class ErrorCodeConfig:
+    SERVICE_PREFIX: int = os.getenv("ERROR_CODE_SERVICE_PREFIX", "0")
+
+
+class ServiceConfig(metaclass=OsAttrMeta):
+    HTTP_PORT: int = 8100
+    TIMEOUT_KEEP_ALIVE: int = 1000
+    PROCESS_NUM: int = 1

algo_backend_framework-0.0.1/algo_backend/config/loguru_config.py

@@ -0,0 +1,19 @@
+from typing import List
+
+from algo_backend.utils import OsAttrMeta
+
+
+class LoguruConfig(metaclass=OsAttrMeta):
+    LOGGER_PATH: str = "/logger"
+    LOG_RETENTION_DAY: int = 60
+    DISABLE_LOG_PKG: str = ""
+    LOG_ADD_CONTAINED_ID: bool = False
+    SAVE_INFO_LEVEL: bool = False
+    SAVE_DEBUG_LOG: bool = True
+
+    @classmethod
+    def get_disable_log_pkg(cls) -> List[str]:
+        if cls.DISABLE_LOG_PKG:
+            return cls.DISABLE_LOG_PKG.split(",")
+        else:
+            return []

algo_backend_framework-0.0.1/algo_backend/exception/__init__.py

@@ -0,0 +1,22 @@
+from .error_code_manage import ApiErrorCodeManage, BasicCodeManage
+from .exception import BasicApiInnerException, BasicCommonException, BasicException
+from .status_code import (
+    BasicApiId,
+    BasicApiInnerErrorCode,
+    BasicStatusCode,
+    CommonStatusCode,
+    DefaultApiErrorCode,
+)
+
+__all__ = [
+    "BasicStatusCode",
+    "CommonStatusCode",
+    "BasicApiId",
+    "BasicApiInnerErrorCode",
+    "BasicApiInnerException",
+    "ApiErrorCodeManage",
+    "BasicCodeManage",
+    "BasicCommonException",
+    "DefaultApiErrorCode",
+    "BasicException",
+]

algo_backend_framework-0.0.1/algo_backend/exception/error_code_manage.py

@@ -0,0 +1,126 @@
+import inspect
+import sys
+from typing import Optional
+
+from algo_backend.config import ErrorCodeConfig
+
+from . import status_code
+from .status_code import BasicApiId, BasicApiInnerErrorCode, BasicStatusCode
+
+
+class ApiErrorCodeManage:
+    """
+    构建”5aaabb“错误码，aaa接口id，bb表示接口内部错误码
+    """
+
+    @classmethod
+    def set_error_code_prefix_env(cls, prefix: Optional[int] = None):
+        """
+        服务启动时执行这个函数
+        也可以启动服务时通过环境变量ERROR_CODE_SERVICE_PREFIX指定服务前缀
+        如果输入的prefix code不符合规则，则忽略，保持原来的6位
+        """
+        if prefix and 1 <= prefix <= 99:
+            ErrorCodeConfig.SERVICE_PREFIX = prefix
+
+    @classmethod
+    def scan_module_and_summary(cls, *list_module) -> list:
+        """扫描模块中的ApiId和错误码枚举类"""
+        api_id_dict: dict = {}
+        api_err_code_info = []
+
+        cls_list = []
+        for m in list_module:
+            cls_list.extend(
+                [obj for name, obj in inspect.getmembers(m) if inspect.isclass(obj)]
+            )
+
+        for obj in set(cls_list):
+            # 查找继承自BasicApiId的枚举类
+            if issubclass(obj, BasicApiId) and obj != BasicApiId:
+                for enum_member in obj:
+                    api_id_dict[enum_member.value] = enum_member.name
+                    api_err_code_info.append(
+                        [
+                            enum_member.value,
+                            BasicApiInnerErrorCode.gen_standard_code(enum_member, 0),
+                            "",
+                        ]
+                    )
+
+            # 查找继承自BasicApiInnerErrorCode的枚举类
+            elif (
+                issubclass(obj, BasicApiInnerErrorCode)
+                and obj != BasicApiInnerErrorCode
+                and hasattr(obj, "__api_id__")
+            ):
+                api_id_enum: BasicApiId = getattr(obj, "__api_id__")
+                for enum_member in obj:
+                    api_err_code_info.append(
+                        [
+                            api_id_enum.value,
+                            BasicApiInnerErrorCode.gen_standard_code(
+                                api_id_enum, enum_member.value
+                            ),
+                            enum_member.msg,
+                        ]
+                    )
+
+        for idx, (api_code, _, _) in enumerate(api_err_code_info):
+            api_err_code_info[idx][0] = api_id_dict[api_code]
+
+        # 排序
+        api_err_code_info.sort(key=lambda x: x[1])
+
+        return api_err_code_info
+
+    @classmethod
+    def summary_module_api_error_code_markdown(cls, *list_module):
+        """
+        打印模块中的ApiId和错误码枚举类
+        """
+        result = cls.scan_module_and_summary(*list_module)
+
+        print("| ApiName | ErrorCode | Message |")
+        print("|--------|------------|---------|")
+        for api_id, api_code, msg in result:
+            print(f"| {api_id} | {api_code} | {msg} |")
+
+
+class BasicCodeManage:
+    __DEFAULT_MODULE = sys.modules[status_code.__name__]
+
+    @classmethod
+    def scan_module_and_summary(cls, *list_module) -> list:
+        """扫描模块中BasicStatusCode枚举类及其子类"""
+        if cls.__DEFAULT_MODULE not in list_module:
+            list_module = list_module + (cls.__DEFAULT_MODULE,)
+
+        result = []
+        cls_list = []
+
+        for m in list_module:
+            cls_list.extend(
+                [obj for name, obj in inspect.getmembers(m) if inspect.isclass(obj)]
+            )
+
+        for obj in set(cls_list):
+            if issubclass(obj, BasicStatusCode) and obj != BasicApiId:
+                for enum_member in obj:
+                    result.append([enum_member.value, enum_member.msg])
+
+        result.sort(key=lambda x: x[0])
+
+        return result
+
+    @classmethod
+    def summary_error_code_markdown(cls, *list_module):
+        """
+        打印模块中的ApiId和错误码枚举类
+        """
+        result = cls.scan_module_and_summary(*list_module)
+
+        print("| Code | Message |")
+        print("|--------|------------|")
+        for code, msg in result:
+            print(f"| {code} | {msg} |")

algo_backend_framework-0.0.1/algo_backend/exception/exception.py

@@ -0,0 +1,42 @@
+from typing import TypeVar
+
+from .status_code import BasicApiId, BasicApiInnerErrorCode, BasicStatusCode
+
+
+class BasicException(Exception):
+    """
+    基础异常类
+    """
+
+    def __init__(self, code: int, msg: str):
+        super().__init__(msg)
+        self.code = code
+        self.msg = msg
+
+
+# 定义泛型变量 T，限定为 BasicStatusCode 的子类
+T = TypeVar("T", bound=BasicStatusCode)
+
+
+class BasicCommonException(BasicException):
+    def __init__(self, status_code: T, **kwargs):
+        super().__init__(code=status_code.code, msg=status_code.msg.format(**kwargs))
+
+
+# 定义泛型变量 ET，限定为 BasicApiInnerErrorCode 的子类
+ET = TypeVar("ET", bound=BasicApiInnerErrorCode)
+
+
+class BasicApiInnerException(BasicException):
+    """
+    接口内部异常类的基类
+    """
+
+    def __init__(self, status_code: ET, **kwargs):
+        super().__init__(code=status_code.code, msg=status_code.msg.format(**kwargs))
+
+    def add_api_id(self, api_id: BasicApiId) -> "BasicApiInnerException":
+        self.code = BasicApiInnerErrorCode.gen_standard_code(
+            api_id=api_id, inner_error_code=self.code
+        )
+        return self

algo_backend_framework-0.0.1/algo_backend/exception/status_code.py

@@ -0,0 +1,103 @@
+from enum import Enum
+
+from algo_backend.config import ErrorCodeConfig
+
+
+def add_service_prefix(v: int):
+    """
+    对于失败的状态码，添加服务号码前缀
+    """
+    v_str = str(v)
+    if len(v_str) == 8:
+        return v
+    if v is not None and v != 0 and ErrorCodeConfig.SERVICE_PREFIX:
+        v = int(f"{ErrorCodeConfig.SERVICE_PREFIX}{v_str}")
+    return v
+
+
+class BasicStatusCode(Enum):
+    def __new__(cls, value: int, msg: str):
+        assert 0 <= value <= 999999, f"错误码长度不能超过6位，且非负数，{value}"
+        obj = object.__new__(cls)
+        obj._value_ = obj
+        obj._code = value
+        obj.msg = msg
+        return obj
+
+    @property
+    def code(self):
+        return add_service_prefix(self._code)
+
+    @property
+    def value(self):
+        return self.code
+
+
+class CommonStatusCode(BasicStatusCode):
+    SUCCESS = (0, "成功")
+
+    ERROR_REQ_URL = (400001, "请求路径错误")
+    ERROR_REQ_METHOD = (400002, "{url}请求方法错误")
+    BODY_EMPTY_ERR = (400003, "{url}请求体内容为空")
+    PARAM_ERROR = (400006, "接口{url}参数错误,{msg}")
+    ERROR_HEADER_NOW_EXISTS = (400007, "接口{url}缺少请求头:{header}")
+
+    UNKNOWN_ERROR = (500000, "未知错误: {msg}")
+
+
+class BasicApiId(Enum):
+    """
+    接口编号枚举类
+    """
+
+    def __new__(cls, value: int):
+        assert 0 <= value <= 999, f"接口编号长度不能超过3位，且非负数，{value}"
+        obj = object.__new__(cls)
+        obj._value_ = obj
+        obj.code = value
+        return obj
+
+    @property
+    def value(self):
+        return self.code
+
+    def bind_api(self):
+        """
+        装饰器，绑定绑定api id
+        """
+
+        def decorator(enum_class):
+            enum_class.__api_id__ = self
+            return enum_class
+
+        return decorator
+
+
+class DefaultApiErrorCode(BasicApiId):
+    """
+    0表示不提供任何ApiId
+    """
+
+    DEFAULT_ERROR = 0
+
+
+class BasicApiInnerErrorCode(Enum):
+    def __new__(cls, value: int, msg: str):
+        assert 1 <= value <= 99, f"接口内部错误码长度不能超过2位，且为正数，{value}"
+        obj = object.__new__(cls)
+        obj._value_ = obj
+        obj.code = value
+        obj.msg = msg
+        return obj
+
+    @property
+    def value(self):
+        return self.code
+
+    @classmethod
+    def gen_standard_code(cls, api_id: BasicApiId, inner_error_code: int = 0):
+        """
+        拼接并生成6位错误码
+        """
+        api_id = api_id or DefaultApiErrorCode.DEFAULT_ERROR
+        return 500000 + api_id.code * 100 + inner_error_code

algo_backend_framework-0.0.1/algo_backend/handler/__init__.py

@@ -0,0 +1,3 @@
+from .operation_handler import timing_and_exception_handler
+
+__all__ = ["timing_and_exception_handler"]

algo_backend_framework-0.0.1/algo_backend/handler/exception_to_vo.py

@@ -0,0 +1,37 @@
+import logging
+from typing import Optional
+
+from algo_backend.exception import (
+    BasicApiId,
+    BasicApiInnerException,
+    BasicCommonException,
+    CommonStatusCode,
+)
+from algo_backend.schema import AbstractRespVo
+
+logger = logging.getLogger(__name__)
+
+
+def gen_vo_from_exception(
+    vo_cls: type(AbstractRespVo),
+    e: Exception,
+    api_name: Optional[str] = None,
+    request_id: Optional[str] = None,
+    api_id: Optional[BasicApiId] = None,
+) -> AbstractRespVo:
+    """
+    将异常转换为vo
+    """
+
+    _params = dict(api_name=api_name, request_id=request_id)
+    if isinstance(e, BasicCommonException):
+        vo = vo_cls.from_exception(e, **_params)
+    elif isinstance(e, BasicApiInnerException):
+        vo = vo_cls.from_exception(e.add_api_id(api_id=api_id), **_params)
+    else:
+        vo = vo_cls.from_exception(
+            BasicCommonException(CommonStatusCode.UNKNOWN_ERROR, msg=str(e)),
+            **_params,
+        )
+
+    return vo

algo_backend_framework-0.0.1/algo_backend/handler/operation_handler.py

@@ -0,0 +1,71 @@
+import logging
+import time
+import traceback
+from functools import wraps
+from typing import Awaitable, Callable, get_type_hints
+
+from algo_backend.exception import (
+    BasicApiId,
+    DefaultApiErrorCode,
+)
+from algo_backend.metrics import PrometheusTimeCostMetricSetting
+from algo_backend.schema import AbstractRespVo
+from .exception_to_vo import gen_vo_from_exception
+
+logger = logging.getLogger(__name__)
+
+
+def timing_and_exception_handler(
+    func=None,
+    *,
+    api_id: BasicApiId = DefaultApiErrorCode.DEFAULT_ERROR,
+    api_name: str = "",
+):
+    """
+    装饰器：用于统计函数执行时间并捕获异常
+    函数中需要包含参数reqid或者request_id
+    """
+
+    def decorator(
+        func: Callable[..., Awaitable[AbstractRespVo]],
+    ) -> Callable[..., Awaitable[AbstractRespVo]]:
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            start_time = time.perf_counter()
+
+            request_id = kwargs.get("request_id", None) or kwargs.get("reqid", None)
+            _name = func.__name__ or api_name
+
+            try:
+                # 执行原函数
+                logger.info(f"ReqId: {request_id} | Function: {_name} | Start")
+                result: AbstractRespVo = await func(*args, **kwargs)
+                # 计算耗时
+                elapsed_time = time.perf_counter() - start_time
+                logger.info(
+                    f"ReqId: {request_id} | Function: {_name} | COST:{elapsed_time:.4f}s"
+                )
+                PrometheusTimeCostMetricSetting.api_metrics_instance().add(
+                    _name, elapsed_time
+                )
+                return result.set_request_id(request_id)
+            except Exception as e:
+                # 计算耗时
+                elapsed_time = time.perf_counter() - start_time
+                PrometheusTimeCostMetricSetting.api_metrics_instance().add_error(_name)
+                # 记录异常信息和完整堆栈
+                logger.error(
+                    f"ReqId: {request_id} | Function: {_name} | COST:{elapsed_time:.4f}s | Exception: {str(e)}\n"
+                    f"Traceback:\n{traceback.format_exc()}"
+                )
+
+                vo_cls: type(AbstractRespVo) = get_type_hints(func).get(
+                    "return"
+                )  # 这里可能会失败，因为无法强制用户的类型
+                return gen_vo_from_exception(
+                    vo_cls, e, api_name=_name, request_id=request_id, api_id=api_id
+                )
+
+        return wrapper
+
+    return decorator if func is None else decorator(func)

algo_backend_framework-0.0.1/algo_backend/intercept/__init__.py

@@ -0,0 +1,9 @@
+from .common import BasicExceptionInterceptor
+from .http import HTTPExceptionInterceptor
+from .validate import ValidateExceptionInterceptor
+
+__all__ = [
+    "HTTPExceptionInterceptor",
+    "ValidateExceptionInterceptor",
+    "BasicExceptionInterceptor",
+]

algo_backend_framework-0.0.1/algo_backend/intercept/common.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Optional
+
+from starlette.requests import Request
+
+from algo_backend.schema import AbstractRespVo, BaseRespVo
+from algo_backend.utils.utils import gen_random_request_id
+
+
+class BasicExceptionInterceptor(ABC):
+    """
+    拦截接口关于schema检验的报错，并返回约定的body
+    """
+
+    def __init__(
+        self,
+        default_vo_type: type(AbstractRespVo) = BaseRespVo,
+        url_vo_dict: Optional[Dict[str, type(AbstractRespVo)]] = None,
+    ):
+        self.default_vo_type: type(AbstractRespVo) = default_vo_type
+        # url和响应的匹配策略
+        self.url_vo_dict = url_vo_dict or {}
+
+    def get_vo_type(self, url: str) -> type(AbstractRespVo):
+        """
+        用户可以继承BasicExceptionInterceptor的子类，覆盖这个方法从而实现更加复杂的url和响应体匹配策略
+        :param url:
+        :return:
+        """
+        return self.url_vo_dict.get(url, self.default_vo_type)
+
+    @classmethod
+    def extract_url(cls, request: Request):
+        try:
+            url = request.url.path
+        except Exception:
+            url = ""
+        return url
+
+    @classmethod
+    def get_request_id(cls, request: Request):
+        return request.headers.get("x-request-id", gen_random_request_id())
+
+    @abstractmethod
+    def intercept(self, request: Request, exc: Exception): ...

algo_backend_framework-0.0.1/algo_backend/intercept/http.py

@@ -0,0 +1,40 @@
+import logging
+
+from fastapi.exceptions import StarletteHTTPException
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+from algo_backend.exception import BasicCommonException, CommonStatusCode
+
+from .common import BasicExceptionInterceptor
+
+logger = logging.getLogger(__name__)
+
+
+class HTTPExceptionInterceptor(BasicExceptionInterceptor):
+    """
+    拦截接口关于http异常的报错，并返回约定的body
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def intercept(self, request: Request, exc: StarletteHTTPException):
+        url: str = self.extract_url(request)
+        reqid: str = self.get_request_id(request)
+
+        logger.info(
+            f"req_id: {url} | 错误码: {exc.status_code} | 错误信息: {exc.detail}"
+        )
+        if exc.status_code == 405:
+            e = BasicCommonException(CommonStatusCode.ERROR_REQ_METHOD, url=url)
+        elif exc.status_code == 404:
+            e = BasicCommonException(CommonStatusCode.ERROR_REQ_URL, url=url)
+        else:
+            e = BasicCommonException(
+                CommonStatusCode.UNKNOWN_ERROR, msg=f"{exc.status_code},{exc.detail}"
+            )
+
+        vo = self.get_vo_type(url).from_exception(e, request_id=reqid)
+
+        return JSONResponse(content=vo.model_dump(), status_code=200)