streamgate 0.1.0__py3-none-any.whl

streamgate/__init__.py

@@ -0,0 +1,214 @@
+"""streamgate 公共 API 出口：管道机制内核（admission + transport + resilience + sink）。
+
+定位：条件接收 → 可靠投递 → 可插拔落地的数据管道框架。
+- 机制归包：准入（唯一性契约：存在性判定+原子占位+冷回源）、背压/测活/自愈
+  （重连、paused、DLQ 二分、位点）、Kafka 收发、批量缓冲、优雅停机
+- 策略归使用方：entity/slot/summary、Schema 校验、落地目标（sink/upserts/on_record）
+- 呈现归使用方：HTTP 路由/鉴权/OpenAPI、健康端点暴露（包内零 fastapi/uvicorn）
+
+Tier 0 声明式（90% 使用方）：IngestBinding + ConsumeSpec/Upsert
+    → IngestGateway（接收内核）/ ConsumerWorker（消费内核）
+Tier 1 组件替换：protocols.py 中的协议 + 下方内置实现
+Tier 2 逃生口：ConsumeSpec.on_record / ConsumeContext（只读快照）
+
+使用方只允许 import 本模块，不得深入内部子模块
+（import-linter 门禁强制）。
+
+依赖分层（机制进核心，策略进 extras）：redis / httpx 是可选策略实现的载体，
+对应导出符号经模块级 __getattr__（PEP 562）惰性装载——裸装可正常 import 本模块；
+访问未安装 extras 的门控符号时抛出带安装指引的 ImportError。
+"""
+
+from importlib import import_module
+from typing import TYPE_CHECKING
+
+from streamgate._optional import require_optional
+from streamgate.config import (
+    BackpressureConfig,
+    ConsumerConfig,
+    DbConfig,
+    KafkaConfig,
+    RedisConfig,
+)
+from streamgate.consumer.classifier import (
+    FailureCategory,
+    add_failure_rule,
+    classify_write_failure,
+)
+from streamgate.consumer.dlq import (
+    BisectOutcome,
+    BufferedMessage,
+    DlqProducer,
+    DlqSendError,
+    QuarantineRequest,
+    locate_and_write,
+)
+from streamgate.consumer.loop import ConsumeRuntime, consume_loop
+from streamgate.consumer.runner import ConsumerWorker
+from streamgate.db.backfill import SqlBackfill
+from streamgate.db.engines import (
+    async_session_factory,
+    create_read_engine,
+    create_write_engine,
+)
+from streamgate.db.upsert import UpsertWriter
+from streamgate.ingest.admission.no_admission import NoAdmission
+from streamgate.ingest.gateway import IngestGateway
+from streamgate.ingest.producer import KafkaProducerService
+from streamgate.obs.logging import configure_logger, logger
+from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink
+from streamgate.protocols import (
+    AdmissionPolicy,
+    AllowAllSignal,
+    BackfillSource,
+    BackpressureSignal,
+    BackpressureSnapshot,
+    ConsumeContext,
+    Decision,
+    DecisionKind,
+    Envelope,
+    IngestOutcome,
+    JsonObject,
+    MessageCodec,
+    NoBackfill,
+    OutcomeKind,
+    ProbeResult,
+    RecordHandler,
+    RecordWriter,
+    RejectInfo,
+    WriteResult,
+)
+from streamgate.resilience.health import (
+    ConsumerHealthResponse,
+    IngestHealthResponse,
+    collect_consumer_health,
+    collect_ingest_health,
+)
+from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT, Upsert
+from streamgate.transport.codec import JsonEnvelopeCodec
+from streamgate.transport.kafka import KafkaConsumerService
+
+if TYPE_CHECKING:
+    # extras 门控符号：运行时经 __getattr__ 惰性装载（见模块 docstring）
+    from streamgate.cache.existence import EMPTY_FIELD, RedisExistenceCache
+    from streamgate.ingest.admission.redis_existence import (
+        EntitySlots,
+        ExistenceUnavailableError,
+        RedisExistenceAdmission,
+        RedisExistenceAdmissionConfig,
+        SlotSource,
+        UndeterminedReason,
+    )
+    from streamgate.resilience.backpressure import HttpProbeSignal
+
+__version__ = "0.1.0"
+
+# extras 门控导出表：符号 → (来源模块, 顶层依赖名)。
+# 裸装访问这些符号时抛 ImportError（含 pip install streamgate[extra] 指引）。
+_EXTRA_EXPORTS: dict[str, tuple[str, str]] = {
+    "EMPTY_FIELD": ("streamgate.cache.existence", "redis"),
+    "RedisExistenceCache": ("streamgate.cache.existence", "redis"),
+    "EntitySlots": ("streamgate.ingest.admission.redis_existence", "redis"),
+    "ExistenceUnavailableError": (
+        "streamgate.ingest.admission.redis_existence",
+        "redis",
+    ),
+    "RedisExistenceAdmission": (
+        "streamgate.ingest.admission.redis_existence",
+        "redis",
+    ),
+    "RedisExistenceAdmissionConfig": (
+        "streamgate.ingest.admission.redis_existence",
+        "redis",
+    ),
+    "SlotSource": ("streamgate.ingest.admission.redis_existence", "redis"),
+    "UndeterminedReason": ("streamgate.ingest.admission.redis_existence", "redis"),
+    "HttpProbeSignal": ("streamgate.resilience.backpressure", "httpx"),
+}
+
+
+def __getattr__(name: str) -> object:
+    """PEP 562 惰性装载：extras 门控符号按需导入，其余符号维持 AttributeError。"""
+    target = _EXTRA_EXPORTS.get(name)
+    if target is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module_name, dependency = target
+    require_optional(dependency)
+    value: object = getattr(import_module(module_name), name)
+    globals()[name] = value  # 缓存：后续访问不再走 __getattr__
+    return value
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)
+
+
+__all__ = [
+    "AdmissionPolicy",
+    "AllowAllSignal",
+    "BackfillSource",
+    "BackpressureConfig",
+    "BackpressureSignal",
+    "BackpressureSnapshot",
+    "BisectOutcome",
+    "EntitySlots",
+    "SlotSource",
+    "BufferedMessage",
+    "ConsumerConfig",
+    "ConsumerHealthResponse",
+    "ConsumerWorker",
+    "ConsumeContext",
+    "ConsumeRuntime",
+    "ConsumeSpec",
+    "DbConfig",
+    "Decision",
+    "DecisionKind",
+    "DlqProducer",
+    "DlqSendError",
+    "EMPTY_FIELD",
+    "Envelope",
+    "ExistenceUnavailableError",
+    "FailureCategory",
+    "HttpProbeSignal",
+    "IngestBinding",
+    "IngestGateway",
+    "IngestHealthResponse",
+    "IngestOutcome",
+    "IngestRecordT",
+    "JsonObject",
+    "JsonEnvelopeCodec",
+    "KafkaConfig",
+    "KafkaConsumerService",
+    "KafkaProducerService",
+    "LoggingMetricsSink",
+    "MessageCodec",
+    "MetricsSink",
+    "NoAdmission",
+    "NoBackfill",
+    "OutcomeKind",
+    "ProbeResult",
+    "QuarantineRequest",
+    "RecordHandler",
+    "RecordWriter",
+    "RedisExistenceAdmission",
+    "RedisExistenceAdmissionConfig",
+    "RedisExistenceCache",
+    "RedisConfig",
+    "RejectInfo",
+    "SqlBackfill",
+    "UndeterminedReason",
+    "Upsert",
+    "UpsertWriter",
+    "WriteResult",
+    "add_failure_rule",
+    "async_session_factory",
+    "classify_write_failure",
+    "collect_consumer_health",
+    "collect_ingest_health",
+    "configure_logger",
+    "consume_loop",
+    "create_read_engine",
+    "create_write_engine",
+    "locate_and_write",
+    "logger",
+]

streamgate/_optional.py

@@ -0,0 +1,38 @@
+"""可选 extras 依赖装载点（惰性导入，C6 门禁基础设施）。
+
+分层约定：redis / httpx 及各类数据库驱动是"可选策略实现"的载体，
+一律不进入模块顶层 import；使用点经本模块校验后再局部导入，
+裸装（无任何 extras）时 `import streamgate` 零第三方可选依赖触达。
+
+对外契约：缺失 extras 时抛出带安装指引的 ImportError
+（而非裸 ModuleNotFoundError），指引文案与 pyproject 的
+[project.optional-dependencies] 保持一致。
+"""
+
+import importlib.util
+from importlib import import_module
+
+# 顶层依赖名 → 安装指引（extra 名与 pyproject 可选依赖组逐字一致）
+_EXTRA_HINTS: dict[str, str] = {
+    "redis": "streamgate[redis]",
+    "httpx": "streamgate[http-probe]",
+    "aiosqlite": "streamgate[sqlite]",
+    "aioodbc": "streamgate[mssql]",
+}
+
+
+def require_optional(dotted: str) -> None:
+    """校验可选依赖可用，缺失即抛 ImportError（含 pip 安装指引）。
+
+    dotted 为目标模块点路径（如 "redis.asyncio"），指引按顶层包名匹配。
+    通过校验后由调用方在局部作用域执行真实 import（保留完整类型推导）。
+    """
+    top = dotted.split(".", 1)[0]
+    if importlib.util.find_spec(top) is None:
+        hint = _EXTRA_HINTS.get(top)
+        guidance = f"Install it with: pip install {hint}" if hint else ""
+        raise ImportError(
+            f"Optional dependency '{top}' is required for this feature but is not "
+            f"installed. {guidance}".rstrip()
+        )
+    import_module(dotted)

streamgate/cache/existence.py

@@ -0,0 +1,233 @@
+"""RedisExistenceCache：entity→slot→summary 的 Redis HASH 缓存（可插拔准入的参考存储）。
+
+机制（摘要结构为 JSON dict）：
+- 原子占位（单 key Lua，Cluster 兼容）：写入即无条件续期回满额（idle GC）
+- 幂等摘要写：无条件 HSET + 续期（TTL 心跳）
+- 空实体哨兵：仅 key 不存在时写入（防与占位竞态）
+- fail-closed 与否是准入策略的参数，缓存层只如实报告错误
+"""
+
+import asyncio
+import json
+from typing import TYPE_CHECKING, TypeVar
+
+from streamgate._optional import require_optional
+from streamgate.config import RedisConfig
+from streamgate.obs.logging import logger
+
+if TYPE_CHECKING:
+    # redis 是 extras 依赖（streamgate[redis]）：仅类型检查可见，运行时惰性装载
+    import redis.asyncio as aioredis
+
+_T = TypeVar("_T")
+
+_HEALTH_TIMEOUT_S = 2.0
+
+# 哨兵 field 保留字：空实体标记。所有 HGETALL/HKEYS 消费者必须过滤它。
+EMPTY_FIELD: str = "__empty__"
+
+# 原子占位（单 key，Cluster 兼容）：写入即无条件续期回满额（idle GC）
+# KEYS[1]=existence key, ARGV=[slot, summary_json, ttl_seconds, empty_field]
+# 返回 [1, ""] = 占位成功；[0, existing_json] = 已被占（竞态输家未写入，不续期）
+_RESERVE_LUA = """
+local existing = redis.call('HGET', KEYS[1], ARGV[1])
+if existing then
+    return {0, existing}
+end
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+redis.call('HDEL', KEYS[1], ARGV[4])
+redis.call('EXPIRE', KEYS[1], ARGV[3])
+return {1, ''}
+"""
+
+# 幂等摘要写（overwrite 路径 / 消费端权威刷新）：无条件 HSET + 无条件续期（idle GC，
+# 消费端权威刷新即 TTL 心跳：key 过期时 DB 几乎必然已权威）
+# KEYS[1]=existence key, ARGV=[slot, summary_json, ttl_seconds, empty_field]
+_SET_SUMMARY_LUA = """
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+redis.call('HDEL', KEYS[1], ARGV[4])
+redis.call('EXPIRE', KEYS[1], ARGV[3])
+return 1
+"""
+
+# 空实体哨兵：仅 key 不存在时写入（哨兵闸门的原子保障，防与占位竞态）
+# KEYS[1]=existence key, ARGV=[empty_ttl_seconds, empty_field]
+_EMPTY_MARKER_LUA = """
+if redis.call('EXISTS', KEYS[1]) == 0 then
+    redis.call('HSET', KEYS[1], ARGV[2], '{"empty":true}')
+    redis.call('EXPIRE', KEYS[1], ARGV[1])
+    return 1
+end
+return 0
+"""
+
+
+def parse_summary(raw: str) -> dict:
+    """解析 field value；损坏数据不致命（返回空摘要，宁可多报 409）。"""
+    try:
+        parsed = json.loads(raw)
+        if isinstance(parsed, dict):
+            return parsed
+    except Exception:
+        pass
+    logger.warning("cache_meta_corrupt", raw=raw[:100])
+    return {}
+
+
+def dumps_summary(summary: dict[str, object]) -> str:
+    return json.dumps(summary, ensure_ascii=False, default=str)
+
+
+class RedisExistenceCache:
+    def __init__(
+        self, config: RedisConfig, client: "aioredis.Redis | None" = None
+    ) -> None:
+        self._config = config
+        self._injected = client
+        self._client: "aioredis.Redis | None" = None
+
+    @property
+    def existence_ttl_seconds(self) -> int:
+        return self._config.existence_ttl_seconds
+
+    async def start(self) -> None:
+        if self._client is not None:
+            return
+        if self._injected is not None:
+            self._client = self._injected
+            return
+        # redis 为 extras 依赖：仅在此真正建连时装载（缺失抛带指引的 ImportError）
+        require_optional("redis.asyncio")
+        import redis.asyncio as aioredis
+
+        self._client = aioredis.from_url(
+            self._config.url,
+            socket_timeout=self._config.socket_timeout_ms / 1000,
+            socket_connect_timeout=self._config.socket_timeout_ms / 1000,
+            decode_responses=True,
+        )
+        logger.info("redis_client_created", url=self._config.url)
+
+    async def close(self) -> None:
+        if self._client is None:
+            return
+        if self._injected is None:  # 注入的客户端归调用方关闭
+            await self._client.aclose()
+        self._client = None
+        logger.info("redis_client_closed")
+
+    async def check_health_detail(self) -> tuple[bool, str | None]:
+        """健康预检。返回 (是否可用, 错误信息)；可用时 error 为 None。"""
+        if self._client is None:
+            return False, "redis client not started"
+        try:
+            await asyncio.wait_for(self._client.ping(), timeout=_HEALTH_TIMEOUT_S)
+            return True, None
+        except Exception as e:
+            logger.debug("redis_health_check_failed", error=str(e))
+            return False, str(e)
+
+    async def check_health(self) -> bool:
+        ok, _ = await self.check_health_detail()
+        return ok
+
+    def existence_key(self, entity: str) -> str:
+        return f"{self._config.key_prefix}existence:{entity}"
+
+    # ---------- 内部工具 ----------
+
+    def _require_client(self) -> "aioredis.Redis":
+        if self._client is None:
+            raise RuntimeError("RedisExistenceCache not started, call start() first")
+        return self._client
+
+    async def _read(self, coro):
+        """读操作统一超时（查询路径 socket_timeout_ms）。"""
+        return await asyncio.wait_for(coro, timeout=self._config.socket_timeout_ms / 1000)
+
+    async def _write(self, coro):
+        """写操作统一超时（接收路径 recv_timeout_ms）。"""
+        return await asyncio.wait_for(coro, timeout=self._config.recv_timeout_ms / 1000)
+
+    # ---------- 读 ----------
+
+    async def get_field_meta(self, entity: str, slot: str) -> dict[str, object] | None:
+        raw = await self._read(self._require_client().hget(self.existence_key(entity), slot))
+        if raw is None:
+            return None
+        return parse_summary(raw)
+
+    async def key_exists(self, entity: str) -> bool:
+        return bool(await self._read(self._require_client().exists(self.existence_key(entity))))
+
+    async def get_entity_fields(self, entity: str) -> dict[str, dict[str, object]] | None:
+        raw: dict[str, str] = await self._read(
+            self._require_client().hgetall(self.existence_key(entity))
+        )
+        if not raw:
+            return None  # Redis 中不存在只有 0 个 field 的 HASH，空 dict 即 key 不存在
+        return {
+            k: parse_summary(v) for k, v in raw.items() if k != EMPTY_FIELD
+        }
+
+    async def get_ttl(self, entity: str) -> int:
+        return int(await self._read(self._require_client().ttl(self.existence_key(entity))))
+
+    # ---------- 写 ----------
+
+    async def reserve_field(
+        self, entity: str, slot: str, summary: dict[str, object]
+    ) -> tuple[bool, dict[str, object] | None]:
+        client = self._require_client()
+        script = client.register_script(_RESERVE_LUA)
+        result = await self._write(
+            script(
+                keys=[self.existence_key(entity)],
+                args=[slot, dumps_summary(summary), self._config.existence_ttl_seconds, EMPTY_FIELD],
+            )
+        )
+        reserved, existing = int(result[0]), result[1]
+        if reserved == 1:
+            return (True, None)
+        return (False, parse_summary(str(existing)))
+
+    async def write_summary(
+        self, entity: str, slot: str, summary: dict[str, object]
+    ) -> None:
+        client = self._require_client()
+        script = client.register_script(_SET_SUMMARY_LUA)
+        await self._write(
+            script(
+                keys=[self.existence_key(entity)],
+                args=[slot, dumps_summary(summary), self._config.existence_ttl_seconds, EMPTY_FIELD],
+            )
+        )
+
+    async def write_entity_fields(
+        self, entity: str, slots: dict[str, dict[str, object]]
+    ) -> None:
+        if not slots:
+            return
+        client = self._require_client()
+        key = self.existence_key(entity)
+        mapping = {slot: dumps_summary(meta) for slot, meta in slots.items()}
+        async with client.pipeline(transaction=False) as pipe:
+            # redis-py 桩的 FieldT 不变量约束无法匹配 Mapping[str, str]（运行时合法）
+            pipe.hset(key, mapping=mapping)  # type: ignore[reportArgumentType]
+            pipe.hdel(key, EMPTY_FIELD)
+            pipe.expire(key, self._config.existence_ttl_seconds)  # 无条件续期
+            await self._write(pipe.execute())
+
+    async def write_empty_marker(self, entity: str) -> bool:
+        client = self._require_client()
+        script = client.register_script(_EMPTY_MARKER_LUA)
+        created = await self._write(
+            script(
+                keys=[self.existence_key(entity)],
+                args=[self._config.empty_existence_ttl_seconds, EMPTY_FIELD],
+            )
+        )
+        return int(created) == 1
+
+    async def delete_entity(self, entity: str) -> None:
+        await self._write(self._require_client().delete(self.existence_key(entity)))

streamgate/config.py

@@ -0,0 +1,150 @@
+"""streamgate 组件配置对象（每组件独立，Spec 内可覆写）。
+
+配置键名与既有环境变量逐字对应
+（KAFKA__* / CONSUMER__* / DB__* / REDIS__* / BACKPRESSURE__*），
+调用方零配置迁移。
+"""
+
+from pydantic import BaseModel, field_validator
+
+
+class ConsumerConfig(BaseModel):
+    # 必填：消费服务装配时校验（缺失即启动失败，不做项目专属默认）
+    group_id: str | None = None
+    batch_size: int = 100
+    batch_timeout_seconds: float = 5.0
+    max_retries: int = 3
+    retry_backoff_base: float = 1.0
+    # 重连/暂停恢复指数退避（对齐 ingest producer 的自愈策略，替代原硬编码 1.0/30.0）
+    reconnect_base_backoff_seconds: float = 1.0
+    reconnect_max_backoff_seconds: float = 30.0
+    max_poll_records: int = 500
+    session_timeout_ms: int = 30000
+    max_poll_interval_ms: int = 300000
+    auto_offset_reset: str = "earliest"
+    backlog_check_interval_seconds: float = 30.0  # 积压时长检查周期
+    # --- DLQ 隔离：消费端坏数据兜底 ---
+    dlq_enabled: bool = True    # 总开关；false=回退 paused 旧行为（紧急逃生门）
+    dlq_send_retries: int = 3   # DLQ 单条发送总尝试次数（含首次；耗尽即批次转 paused）
+
+
+class KafkaConfig(BaseModel):
+    bootstrap_servers: str = "kafka:9092"
+    # 必填：producer/consumer 装配时校验（缺失即启动失败，不做项目专属默认）
+    topic: str | None = None
+    # 死信 topic（消费端隔离坏数据留档）；启用 DLQ 时必填
+    dlq_topic: str | None = None
+    acks: str = "all"
+    request_timeout_ms: int = 10000
+    enable_idempotence: bool = True
+    # 自愈监控：周期性探活，不健康时销毁旧实例重建连接（覆盖启动期/运行期断线）
+    health_check_interval_seconds: float = 30.0   # 探活轮询周期
+    reconnect_base_backoff_seconds: float = 1.0   # 重连指数退避基数
+    reconnect_max_backoff_seconds: float = 30.0   # 重连退避上限
+    # R1 防抖：连续 send 失败 N 次才触发重建（默认 1=首次失败即重建；
+    # broker 秒级抖动时可调高减少误重建）
+    reconnect_failure_threshold: int = 1
+    # 探活口径：近窗口内有 send 失败即判定不健康（秒）
+    send_failure_window_seconds: float = 30.0
+    # R3 周期分级：异常/重建中高频探活间隔（秒）；健康态仍用 health_check_interval_seconds
+    unhealthy_check_interval_seconds: float = 5.0
+
+    @field_validator("bootstrap_servers", mode="before")
+    @classmethod
+    def strip_url_scheme(cls, v: object) -> object:
+        if not isinstance(v, str):
+            return v
+        cleaned: list[str] = []
+        for part in v.split(","):
+            part = part.strip()
+            for scheme in ("http://", "https://", "kafka://"):
+                if part.lower().startswith(scheme):
+                    part = part[len(scheme):]
+                    break
+            cleaned.append(part)
+        return ",".join(cleaned)
+
+
+class DbConfig(BaseModel):
+    # 必填：声明 upserts/SqlBackfill 时在引擎创建单点校验
+    # （缺失即装配失败，不做项目专属默认；纯 Kafka 管道无需配置）。
+    # 驱动按连接串动态加载：sqlite 需 extras [sqlite]，mssql 需 extras [mssql]。
+    connection_string: str | None = None
+    echo: bool = False
+    # --- L1 快速失败边界（异步落库链路加固）---
+    query_timeout_seconds: int = 5      # existence 回源驱动语句超时（MSSQL；须 <= query_wait_seconds）
+    query_wait_seconds: int = 8         # existence 回源调用层 wait_for
+    write_timeout_seconds: int = 20     # write_batch 驱动语句超时（MSSQL；须 <= write_wait_seconds）
+    write_wait_seconds: int = 25        # write_batch 调用层 wait_for
+    pool_timeout_seconds: int = 3       # 连接池获取超时（池耗尽快速失败）
+    cold_path_max_concurrency: int = 10  # 冷实体回源并发闸门（<=0 禁用；与读池宽度一致留余量）
+    cold_path_gate_retry_after_seconds: int = 1  # 闸门满时建议调用方的重试间隔（秒）
+    # 读连接池（ingest 进程 existence 回源专用）：闸门默认 10，池总容量 20 留余量，
+    # 避免 check_health 等非回源占用与回源争抢触发 pool_timeout 快速失败。
+    read_pool_size: int = 10            # 读池固定连接数
+    read_pool_max_overflow: int = 10    # 读池溢出连接数（峰值余量）
+    # 写连接池（consumer 进程 write_batch 专用）：默认值即原硬编码值（零行为变更）
+    write_pool_size: int = 10           # 写池固定连接数
+    write_pool_max_overflow: int = 20   # 写池溢出连接数
+
+    def require_connection_string(self) -> str:
+        """连接串装配校验（引擎创建单点调用）：None 即配置错误，含修复指引。"""
+        if self.connection_string is None:
+            raise ValueError(
+                "db connection string is required when upserts or SqlBackfill are "
+                "declared: set DbConfig.connection_string (env: DB__CONNECTION_STRING), "
+                "e.g. 'sqlite+aiosqlite:///./data/streamgate.db'"
+            )
+        return self.connection_string
+
+    @property
+    def dialect(self) -> str:
+        """驱动方言标签（日志/分支用）：sqlite | mssql。"""
+        return "sqlite" if "sqlite" in self.require_connection_string().lower() else "mssql"
+
+    @property
+    def redacted_connection_string(self) -> str:
+        """隐藏密码后的连接串（日志用，避免凭据入日志）。"""
+        s = self.require_connection_string()
+        if "://" not in s or "@" not in s:
+            return s
+        scheme, _, rest = s.partition("://")
+        userinfo, _, host = rest.rpartition("@")
+        if ":" in userinfo:
+            user, _, _ = userinfo.partition(":")
+            userinfo = f"{user}:***"
+        return f"{scheme}://{userinfo}@{host}"
+
+
+class RedisConfig(BaseModel):
+    url: str = "redis://localhost:6379/0"
+    key_prefix: str = "streamgate:"
+    existence_ttl_seconds: int = 18000       # existence TTL 5h
+    empty_existence_ttl_seconds: int = 3600  # 空实体哨兵 TTL（短于 existence TTL）
+    socket_timeout_ms: int = 1000      # 查询/校验路径超时
+    recv_timeout_ms: int = 500         # 接收路径占位/摘要写超时
+    # ingest 自身 Redis 不可用时 fail-closed：校验路径/overwrite 路径/位置查询端点
+    # 全部返回错误而非静默降级（杜绝无占位接受与不完整查询结果）；false=旧降级逃生门
+    fail_closed_on_unavailable: bool = True
+
+
+class BackpressureConfig(BaseModel):
+    """ingest 背压拒绝配置（consumption-backpressure）。
+
+    默认值与 existence TTL 联动：trip = existence_ttl/2 = 9000s（漏报窗口打开前拦截，
+    与积压告警线 backlog_age_warn 一致）；recover = trip*80% = 7200s（磁滞防抖）。
+    """
+
+    enabled: bool = True                                # 总开关；False=完全放行（紧急回滚）
+    consumer_health_url: str = "http://localhost:9109/health"  # consumer 健康接口地址
+    check_interval_seconds: float = 30.0                # 轮询周期（与积压检查周期一致）
+    timeout_seconds: float = 2.0                        # 单次探活超时
+    probe_retries: int = 3                              # 单周期内探活重试次数（不含首次）；0=禁用
+    probe_retry_interval_seconds: float = 15.0          # 重试间隔（秒）
+    trip_seconds: float = 9000.0                        # 触发阈值：积压超此值开始拒绝
+    recover_seconds: float = 7200.0                     # 恢复阈值：积压回落至此值以下放行（磁滞）
+    retry_after_seconds: int = 60                       # 拒绝时 Retry-After 头（秒）
+    fail_closed_on_unreachable: bool = True             # 探活不可达时是否拒绝（fail-closed）
+    reject_on_any_degraded: bool = False                # 旧行为逃生门：任何 degraded 即拒绝（恢复全量拒绝语义）
+    # R3 背压周期分级：REJECTING 态高频探活间隔（秒）；OPEN 态仍用 check_interval_seconds
+    unhealthy_check_interval_seconds: float = 5.0