aury-boot 0.0.29__py3-none-any.whl → 0.0.31__py3-none-any.whl

aury/boot/infrastructure/monitoring/tracing/processor.py

@@ -0,0 +1,357 @@
+"""自定义 SpanProcessor，用于检测慢 span 和异常 span 并触发告警。"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+import fnmatch
+import re
+from typing import TYPE_CHECKING, Any
+
+from aury.boot.common.logging import logger
+
+# OTel 可选依赖
+try:
+    from opentelemetry.trace import SpanKind as OTelSpanKind
+    from opentelemetry.trace import StatusCode
+except ImportError:
+    OTelSpanKind = None  # type: ignore[assignment, misc]
+    StatusCode = None  # type: ignore[assignment, misc]
+
+if TYPE_CHECKING:
+    from asyncio import Task
+
+    from opentelemetry.sdk.trace import ReadableSpan, Span, SpanProcessor
+
+# 告警回调类型: async def callback(event_type, message, **metadata)
+AlertCallback = Callable[[str, str], Awaitable[None]] | Callable[..., Awaitable[None]]
+
+
+class AlertingSpanProcessor:
+    """告警 SpanProcessor。
+    
+    在 span 结束时检测：
+    - 慢 span（duration > threshold）
+    - 异常 span（status = ERROR）
+    
+    通过回调函数触发告警，不直接依赖 alerting 模块。
+    
+    注意：这是一个同步 SpanProcessor，告警发送会在后台异步执行。
+    """
+    
+    # 保存后台任务引用，避免被 GC 回收
+    _background_tasks: set["Task"] = set()
+    
+    def __init__(
+        self,
+        *,
+        slow_request_threshold: float = 1.0,
+        slow_sql_threshold: float = 0.5,
+        alert_on_slow_request: bool = True,
+        alert_on_slow_sql: bool = True,
+        alert_on_error: bool = True,
+        alert_callback: AlertCallback | None = None,
+        slow_request_exclude_paths: list[str] | None = None,
+    ) -> None:
+        """初始化 AlertingSpanProcessor。
+        
+        Args:
+            slow_request_threshold: HTTP 请求慢阈值（秒）
+            slow_sql_threshold: SQL 查询慢阈值（秒）
+            alert_on_slow_request: 是否对慢 HTTP 请求发送告警
+            alert_on_slow_sql: 是否对慢 SQL 发送告警
+            alert_on_error: 是否对异常 span 发送告警
+            alert_callback: 告警回调函数，签名: async (event_type, message, **metadata) -> None
+            slow_request_exclude_paths: 慢请求排除路径列表（支持 * 通配符），如 SSE/WebSocket 长连接
+        """
+        self._slow_request_threshold = slow_request_threshold
+        self._slow_sql_threshold = slow_sql_threshold
+        self._alert_on_slow_request = alert_on_slow_request
+        self._alert_on_slow_sql = alert_on_slow_sql
+        self._alert_on_error = alert_on_error
+        self._alert_callback = alert_callback
+        
+        # 编译排除路径正则
+        self._exclude_regexes: list[re.Pattern] = []
+        if slow_request_exclude_paths:
+            for pattern in slow_request_exclude_paths:
+                regex_pattern = fnmatch.translate(pattern)
+                self._exclude_regexes.append(re.compile(regex_pattern))
+    
+    def on_start(self, span: "Span", parent_context: object = None) -> None:
+        """span 开始时调用（不做处理）。"""
+        pass
+    
+    def on_end(self, span: "ReadableSpan") -> None:
+        """span 结束时调用，检测并触发告警。"""
+        if StatusCode is None:
+            return
+        
+        # 获取 span 信息
+        name = span.name
+        duration_ns = (span.end_time or 0) - (span.start_time or 0)
+        duration_s = duration_ns / 1e9
+        status = span.status
+        trace_id = format(span.context.trace_id, "032x") if span.context else ""
+        
+        # 获取 span 属性
+        attributes = dict(span.attributes) if span.attributes else {}
+        
+        # 判断 span 类型
+        span_kind = _get_span_kind(span)
+        
+        # 根据 span 类型获取对应的慢阈值和开关
+        threshold = self._get_slow_threshold(span_kind)
+        should_alert = self._should_alert_slow(span_kind)
+        
+        # 检测慢 span
+        if should_alert and threshold > 0 and duration_s >= threshold:
+            # 检查是否在排除路径中
+            if self._is_path_excluded(name, attributes):
+                return
+            
+            self._emit_slow_alert(
+                name=name,
+                duration=duration_s,
+                trace_id=trace_id,
+                span_kind=span_kind,
+                attributes=attributes,
+                threshold=threshold,
+            )
+        
+        # 检测异常 span（只对 SERVER span 发告警，避免重复）
+        if (
+            self._alert_on_error
+            and status
+            and status.status_code == StatusCode.ERROR
+            and span_kind == "http"  # 只对 HTTP SERVER span 发异常告警
+        ):
+            # 过滤 4xx 业务异常（如 401/403/404 等），只对 5xx 系统异常告警
+            http_status = attributes.get("http.status_code", 0)
+            if 400 <= http_status < 500:
+                return  # 业务异常不告警
+            
+            # 从 span events 中提取异常详情
+            exception_info = _extract_exception_info(span)
+            self._emit_error_alert(
+                name=name,
+                duration=duration_s,
+                trace_id=trace_id,
+                span_kind=span_kind,
+                error_message=exception_info.get("message") or status.description or "Unknown error",
+                attributes=attributes,
+                exception_info=exception_info,
+            )
+    
+    def shutdown(self) -> None:
+        """关闭处理器。"""
+        pass
+    
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """强制刷新（无缓冲，直接返回）。"""
+        return True
+    
+    def _get_slow_threshold(self, span_kind: str) -> float:
+        """根据 span 类型获取对应的慢阈值。"""
+        if span_kind == "database":
+            return self._slow_sql_threshold
+        elif span_kind in ("http", "http_client"):
+            return self._slow_request_threshold
+        # 其他类型使用 HTTP 阈值作为默认
+        return self._slow_request_threshold
+    
+    def _should_alert_slow(self, span_kind: str) -> bool:
+        """根据 span 类型判断是否应该发送慢告警。"""
+        if span_kind == "database":
+            return self._alert_on_slow_sql
+        elif span_kind in ("http", "http_client", "internal"):
+            return self._alert_on_slow_request
+        # 其他类型默认使用 HTTP 的开关
+        return self._alert_on_slow_request
+    
+    def _is_path_excluded(self, name: str, attributes: dict) -> bool:
+        """检查路径是否在排除列表中。"""
+        if not self._exclude_regexes:
+            return False
+        
+        # 从 attributes 或 span name 中提取路径
+        path = (
+            attributes.get("http.route")
+            or attributes.get("http.target")
+            or name
+        )
+        
+        return any(regex.match(path) for regex in self._exclude_regexes)
+    
+    def _emit_slow_alert(
+        self,
+        name: str,
+        duration: float,
+        trace_id: str,
+        span_kind: str,
+        attributes: dict,
+        threshold: float,
+    ) -> None:
+        """发送慢 span 告警。"""
+        if not self._alert_callback:
+            return
+        
+        try:
+            task = asyncio.create_task(
+                self._alert_callback(
+                    _get_event_type_for_slow(span_kind),
+                    f"慢 {span_kind}: {name}",
+                    severity="warning",
+                    trace_id=trace_id,
+                    source=span_kind,
+                    duration=duration,
+                    threshold=threshold,
+                    **_extract_alert_context(attributes),
+                )
+            )
+            self._background_tasks.add(task)
+            task.add_done_callback(self._background_tasks.discard)
+        except RuntimeError:
+            logger.debug(f"无法发送慢 span 告警（无事件循环）: {name}")
+    
+    def _emit_error_alert(
+        self,
+        name: str,
+        duration: float,
+        trace_id: str,
+        span_kind: str,
+        error_message: str,
+        attributes: dict,
+        exception_info: dict | None = None,
+    ) -> None:
+        """发送异常 span 告警。"""
+        if not self._alert_callback:
+            return
+        
+        # 合并异常信息
+        extra_context = _extract_alert_context(attributes)
+        if exception_info:
+            if exception_info.get("type"):
+                extra_context["error_type"] = exception_info["type"]
+            if exception_info.get("stacktrace"):
+                extra_context["stacktrace"] = exception_info["stacktrace"]
+        
+        try:
+            task = asyncio.create_task(
+                self._alert_callback(
+                    "exception",
+                    f"异常: {error_message}",
+                    severity="error",
+                    trace_id=trace_id,
+                    source=span_kind,
+                    duration=duration,
+                    error_message=error_message,
+                    **extra_context,
+                )
+            )
+            self._background_tasks.add(task)
+            task.add_done_callback(self._background_tasks.discard)
+        except RuntimeError:
+            logger.debug(f"无法发送异常 span 告警（无事件循环）: {name}")
+
+
+def _get_span_kind(span: "ReadableSpan") -> str:
+    """根据 span 属性推断类型。"""
+    if OTelSpanKind is None:
+        return "unknown"
+    
+    kind = span.kind
+    
+    if kind == OTelSpanKind.SERVER:
+        # HTTP 请求
+        return "http"
+    elif kind == OTelSpanKind.CLIENT:
+        # 外部调用（DB、HTTP client 等）
+        attributes = dict(span.attributes) if span.attributes else {}
+        if "db.system" in attributes:
+            return "database"
+        elif "http.url" in attributes or "http.target" in attributes:
+            return "http_client"
+        return "client"
+    elif kind == OTelSpanKind.INTERNAL:
+        return "internal"
+    elif kind == OTelSpanKind.PRODUCER:
+        return "producer"
+    elif kind == OTelSpanKind.CONSUMER:
+        return "consumer"
+    
+    return "unknown"
+
+
+def _get_event_type_for_slow(span_kind: str) -> str:
+    """根据 span 类型获取慢操作的告警事件类型。"""
+    mapping = {
+        "http": "slow_request",
+        "database": "slow_sql",
+        "http_client": "slow_request",
+        "internal": "slow_request",  # internal span 也用 slow_request 类型
+    }
+    return mapping.get(span_kind, "custom")
+
+
+def _extract_exception_info(span: "ReadableSpan") -> dict:
+    """从 span events 中提取异常信息。
+    
+    OTEL 会将异常作为 span event 记录，包含：
+    - exception.type: 异常类型
+    - exception.message: 异常消息
+    - exception.stacktrace: 堆栈信息
+    """
+    info = {}
+    
+    if not span.events:
+        return info
+    
+    for event in span.events:
+        if event.name == "exception":
+            attrs = dict(event.attributes) if event.attributes else {}
+            if "exception.type" in attrs:
+                info["type"] = str(attrs["exception.type"])
+            if "exception.message" in attrs:
+                info["message"] = str(attrs["exception.message"])
+            if "exception.stacktrace" in attrs:
+                info["stacktrace"] = str(attrs["exception.stacktrace"])
+            break  # 只取第一个异常事件
+    
+    return info
+
+
+def _extract_alert_context(attributes: dict) -> dict:
+    """从 span 属性中提取告警上下文。"""
+    context = {}
+    
+    # HTTP 相关
+    if "http.method" in attributes:
+        context["method"] = attributes["http.method"]
+    if "http.route" in attributes:
+        context["route"] = attributes["http.route"]
+    if "http.target" in attributes:
+        context["endpoint"] = attributes["http.target"]
+    if "http.url" in attributes:
+        context["url"] = attributes["http.url"]
+    if "http.status_code" in attributes:
+        context["status_code"] = attributes["http.status_code"]
+    
+    # 数据库相关
+    if "db.system" in attributes:
+        context["db_system"] = attributes["db.system"]
+    if "db.statement" in attributes:
+        context["sql"] = _normalize_sql(str(attributes["db.statement"]))
+    
+    return context
+
+
+def _normalize_sql(sql: str) -> str:
+    """清理 SQL 多余空白。"""
+    import re
+    # 将多个空白字符（包括换行、制表符）合并为单个空格
+    sql = re.sub(r'\s+', ' ', sql)
+    return sql.strip()
+
+
+__all__ = ["AlertingSpanProcessor"]

aury/boot/infrastructure/monitoring/tracing/provider.py

@@ -0,0 +1,322 @@
+"""OpenTelemetry TracerProvider 配置和初始化。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from aury.boot.common.logging import logger
+
+from .processor import AlertingSpanProcessor
+
+# OTel 核心可选依赖
+try:
+    from opentelemetry import trace as otel_trace
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.sdk.trace import TracerProvider as OTelTracerProvider
+except ImportError:
+    otel_trace = None  # type: ignore[assignment]
+    Resource = None  # type: ignore[assignment, misc]
+    OTelTracerProvider = None  # type: ignore[assignment, misc]
+
+# OTel Traces 导出可选依赖
+try:
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+except ImportError:
+    OTLPSpanExporter = None  # type: ignore[assignment, misc]
+    BatchSpanProcessor = None  # type: ignore[assignment, misc]
+
+# OTel Metrics 导出可选依赖
+try:
+    from opentelemetry import metrics as otel_metrics
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+except ImportError:
+    otel_metrics = None  # type: ignore[assignment]
+    OTLPMetricExporter = None  # type: ignore[assignment, misc]
+    MeterProvider = None  # type: ignore[assignment, misc]
+    PeriodicExportingMetricReader = None  # type: ignore[assignment, misc]
+
+# OTel Instrumentation 可选依赖
+try:
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+except ImportError:
+    FastAPIInstrumentor = None  # type: ignore[assignment, misc]
+
+try:
+    from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+except ImportError:
+    SQLAlchemyInstrumentor = None  # type: ignore[assignment, misc]
+
+try:
+    from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+except ImportError:
+    HTTPXClientInstrumentor = None  # type: ignore[assignment, misc]
+
+if TYPE_CHECKING:
+    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+
+
+@dataclass
+class TelemetryConfig:
+    """遍测配置。"""
+    
+    # 基础配置
+    service_name: str = "aury-service"
+    service_version: str = ""
+    environment: str = "development"
+    
+    # 启用的 instrumentation
+    instrument_fastapi: bool = True
+    instrument_sqlalchemy: bool = True
+    instrument_httpx: bool = True
+    
+    # 告警配置
+    alert_enabled: bool = True
+    slow_request_threshold: float = 1.0  # HTTP 请求慢阈值（秒）
+    slow_sql_threshold: float = 0.5  # SQL 查询慢阈值（秒）
+    alert_on_slow_request: bool = True  # 是否对慢 HTTP 请求发送告警
+    alert_on_slow_sql: bool = True  # 是否对慢 SQL 发送告警
+    alert_on_error: bool = True
+    alert_callback: Any = None  # 告警回调函数
+    slow_request_exclude_paths: list[str] = field(default_factory=list)  # 慢请求排除路径
+    
+    # OTLP Traces 导出配置
+    traces_endpoint: str | None = None
+    traces_headers: dict[str, str] = field(default_factory=dict)
+    
+    # OTLP Logs 导出配置
+    logs_endpoint: str | None = None
+    logs_headers: dict[str, str] = field(default_factory=dict)
+    
+    # OTLP Metrics 导出配置
+    metrics_endpoint: str | None = None
+    metrics_headers: dict[str, str] = field(default_factory=dict)
+    
+    # 采样配置
+    sampling_rate: float = 1.0  # 1.0 = 100%
+
+
+class TelemetryProvider:
+    """遥测提供者。
+    
+    封装 OpenTelemetry TracerProvider 的配置和初始化逻辑。
+    """
+    
+    def __init__(self, config: TelemetryConfig) -> None:
+        """初始化 TelemetryProvider。
+        
+        Args:
+            config: 遥测配置
+        """
+        self._config = config
+        self._provider: "TracerProvider | None" = None
+        self._initialized = False
+    
+    def initialize(self) -> bool:
+        """初始化 OpenTelemetry。
+        
+        Returns:
+            bool: 是否成功初始化
+        """
+        if self._initialized:
+            return True
+        
+        if OTelTracerProvider is None:
+            logger.warning("OpenTelemetry 初始化失败（缺少依赖）")
+            return False
+        
+        try:
+            # 创建 Resource
+            resource = Resource.create({
+                "service.name": self._config.service_name,
+                "service.version": self._config.service_version,
+                "deployment.environment": self._config.environment,
+            })
+            
+            # 创建 TracerProvider
+            self._provider = OTelTracerProvider(resource=resource)
+            
+            # 添加 SpanProcessor
+            self._setup_processors()
+            
+            # 设置为全局 TracerProvider
+            otel_trace.set_tracer_provider(self._provider)
+            
+            # 配置 instrumentation
+            self._setup_instrumentations()
+            
+            self._initialized = True
+            logger.info(
+                f"OpenTelemetry 初始化完成: "
+                f"service={self._config.service_name}, "
+                f"alert_enabled={self._config.alert_enabled}"
+            )
+            return True
+        except Exception as e:
+            logger.error(f"OpenTelemetry 初始化失败: {e}")
+            return False
+    
+    def shutdown(self) -> None:
+        """关闭 OpenTelemetry。"""
+        if self._provider:
+            try:
+                self._provider.shutdown()
+                logger.info("OpenTelemetry 已关闭")
+            except Exception as e:
+                logger.warning(f"OpenTelemetry 关闭失败: {e}")
+    
+    def _setup_processors(self) -> None:
+        """设置 SpanProcessor。"""
+        if not self._provider:
+            return
+        
+        # 添加告警处理器
+        if self._config.alert_enabled:
+            alerting_processor = AlertingSpanProcessor(
+                slow_request_threshold=self._config.slow_request_threshold,
+                slow_sql_threshold=self._config.slow_sql_threshold,
+                alert_on_slow_request=self._config.alert_on_slow_request,
+                alert_on_slow_sql=self._config.alert_on_slow_sql,
+                alert_on_error=self._config.alert_on_error,
+                alert_callback=self._config.alert_callback,
+                slow_request_exclude_paths=self._config.slow_request_exclude_paths or None,
+            )
+            self._provider.add_span_processor(alerting_processor)
+            logger.debug("已添加 AlertingSpanProcessor")
+        
+        # 添加 OTLP Traces 导出器
+        if self._config.traces_endpoint:
+            self._setup_traces_exporter()
+        
+        # 添加 OTLP Metrics 导出器
+        if self._config.metrics_endpoint:
+            self._setup_metrics_exporter()
+    
+    def _setup_traces_exporter(self) -> None:
+        """设置 Traces OTLP 导出器。"""
+        if not self._provider or not self._config.traces_endpoint:
+            return
+        
+        if OTLPSpanExporter is None:
+            logger.warning("Traces OTLP 导出器未安装，跳过配置")
+            return
+        
+        try:
+            exporter = OTLPSpanExporter(
+                endpoint=self._config.traces_endpoint,
+                headers=self._config.traces_headers or None,
+            )
+            self._provider.add_span_processor(BatchSpanProcessor(exporter))
+            logger.info(f"Traces OTLP 导出器已配置: {self._config.traces_endpoint}")
+        except Exception as e:
+            logger.warning(f"Traces OTLP 导出器配置失败: {e}")
+    
+    def _setup_metrics_exporter(self) -> None:
+        """设置 Metrics OTLP 导出器。"""
+        if not self._config.metrics_endpoint:
+            return
+        
+        if OTLPMetricExporter is None:
+            logger.warning("Metrics OTLP 导出器未安装，跳过配置")
+            return
+        
+        try:
+            resource = Resource.create({
+                "service.name": self._config.service_name,
+                "service.version": self._config.service_version,
+                "deployment.environment": self._config.environment,
+            })
+            
+            exporter = OTLPMetricExporter(
+                endpoint=self._config.metrics_endpoint,
+                headers=self._config.metrics_headers or None,
+            )
+            reader = PeriodicExportingMetricReader(exporter)
+            meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
+            otel_metrics.set_meter_provider(meter_provider)
+            
+            logger.info(f"Metrics OTLP 导出器已配置: {self._config.metrics_endpoint}")
+        except Exception as e:
+            logger.warning(f"Metrics OTLP 导出器配置失败: {e}")
+    
+    def _setup_instrumentations(self) -> None:
+        """配置自动 instrumentation。
+        
+        注意：
+        - FastAPI instrumentation 需要通过 instrument_fastapi_app() 单独调用。
+        - SQLAlchemy instrumentation 需要在 engine 创建后调用（由 DatabaseComponent 处理）。
+        """
+        enabled = []
+        pending = []
+        
+        # SQLAlchemy instrumentation 需要 engine，在 DatabaseComponent 中处理
+        if self._config.instrument_sqlalchemy:
+            pending.append("SQLAlchemy")
+        
+        # HTTPX instrumentation
+        if self._config.instrument_httpx:
+            if self._instrument_httpx():
+                enabled.append("HTTPX")
+        
+        if enabled:
+            logger.info(f"Instrumentation 已启用: {', '.join(enabled)}")
+        if pending:
+            logger.debug(f"Instrumentation 待启用（需要 engine）: {', '.join(pending)}")
+    
+    def instrument_fastapi_app(self, app) -> None:
+        """对已创建的 FastAPI app 进行 instrumentation。
+        
+        Args:
+            app: FastAPI 应用实例
+        """
+        if FastAPIInstrumentor is None:
+            logger.debug("FastAPI instrumentation 未安装")
+            return
+        
+        if not self._config.instrument_fastapi:
+            return
+        
+        try:
+            FastAPIInstrumentor.instrument_app(app)
+            logger.info("FastAPI instrumentation 已启用")
+        except Exception as e:
+            logger.warning(f"FastAPI instrumentation 配置失败: {e}")
+    
+    
+    def _instrument_httpx(self) -> bool:
+        """配置 HTTPX instrumentation。"""
+        if HTTPXClientInstrumentor is None:
+            logger.debug("HTTPX instrumentation 未安装")
+            return False
+        
+        try:
+            HTTPXClientInstrumentor().instrument()
+            return True
+        except Exception as e:
+            logger.warning(f"HTTPX instrumentation 配置失败: {e}")
+            return False
+    
+    def add_processor(self, processor: "SpanProcessor") -> None:
+        """添加自定义 SpanProcessor。
+        
+        Args:
+            processor: SpanProcessor 实例
+        """
+        if self._provider:
+            self._provider.add_span_processor(processor)
+    
+    @property
+    def provider(self) -> "TracerProvider | None":
+        """获取 TracerProvider 实例。"""
+        return self._provider
+    
+    @property
+    def is_initialized(self) -> bool:
+        """是否已初始化。"""
+        return self._initialized
+
+
+__all__ = ["TelemetryConfig", "TelemetryProvider"]