streamgate 0.1.0__py3-none-any.whl

streamgate/consumer/runner.py

@@ -0,0 +1,233 @@
+"""消费服务装配与生命周期（机制内核，无 HTTP 呈现）。
+
+职责：init sink（可选）→ 启动缓存/consumer/DLQ → 消费循环 →
+优雅停机（flush 缓冲、leave group、关资源）→ 健康快照。
+健康数据经 health_snapshot() 暴露；端点实现归使用方。
+"""
+
+import asyncio
+import signal
+
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+)
+
+from streamgate.cache.existence import RedisExistenceCache
+from streamgate.config import ConsumerConfig, DbConfig, KafkaConfig, RedisConfig
+from streamgate.consumer.dlq import DlqProducer
+from streamgate.consumer.loop import ConsumeRuntime, consume_loop
+from streamgate.db.engines import async_session_factory, create_write_engine
+from streamgate.db.upsert import UpsertWriter
+from streamgate.obs.logging import logger
+from streamgate.protocols import MessageCodec, RecordWriter
+from streamgate.resilience.health import (
+    ConsumerHealthResponse,
+    collect_consumer_health,
+)
+from streamgate.specs import ConsumeSpec
+from streamgate.transport.codec import JsonEnvelopeCodec
+from streamgate.transport.kafka import KafkaConsumerService
+
+
+class ConsumerWorker:
+    def __init__(
+        self,
+        spec: ConsumeSpec,
+        *,
+        kafka_config: KafkaConfig,
+        consumer_config: ConsumerConfig,
+        db_config: DbConfig | None = None,
+        redis_config: RedisConfig | None = None,
+        cache: RedisExistenceCache | None = None,
+        writer: RecordWriter | None = None,
+        engine: AsyncEngine | None = None,
+        session_factory: async_sessionmaker[AsyncSession] | None = None,
+        codec: MessageCodec | None = None,
+        existence_ttl_seconds: int | None = None,
+    ) -> None:
+        self.spec = spec
+        self._kafka_config = kafka_config
+        self._consumer_config = consumer_config
+        self._db_config = db_config
+        self._redis_config = redis_config
+        self._cache = cache
+        self._writer = writer
+        self._engine = engine
+        self._session_factory = session_factory
+        self._codec = codec or JsonEnvelopeCodec()
+        self._existence_ttl_seconds = (
+            existence_ttl_seconds
+            if existence_ttl_seconds is not None
+            else (redis_config.existence_ttl_seconds if redis_config is not None else 18000)
+        )
+        self.runtime: ConsumeRuntime | None = None
+        self._stop_event: asyncio.Event | None = None
+
+    # ---- 装配 ----
+
+    def _resolve_writer(self) -> RecordWriter | None:
+        """落地目标解析：显式 sink 优先；upserts 非空时构建内置 UpsertWriter；
+        on_record 模式返回 None（无写侧）。"""
+        if self._writer is not None:
+            return self._writer
+        if not self.spec.upserts:
+            return None
+        if self._db_config is None:
+            raise ValueError(
+                "db_config is required when ConsumeSpec.upserts is set"
+            )
+        engine = self._engine or create_write_engine(self._db_config)
+        factory = self._session_factory or async_session_factory(engine)
+        return UpsertWriter(self._db_config, engine, factory, self.spec.upserts)
+
+    def _build_dlq(self) -> DlqProducer | None:
+        """DLQ 隔离 producer（启动失败仅记日志，懒启动在 quarantine 兜底）。"""
+        enabled = (
+            self.spec.dlq
+            if self.spec.dlq is not None
+            else self._consumer_config.dlq_enabled
+        )
+        if not enabled:
+            return None
+        return DlqProducer(
+            self._kafka_config,
+            topic=self.spec.dlq_topic or None,
+            send_retries=self._consumer_config.dlq_send_retries,
+            message_type=self.spec.dlq_message_type,
+        )
+
+    async def prepare(self) -> ConsumeRuntime:
+        """初始化全部组件（幂等入口）。"""
+        if self.runtime is not None:
+            return self.runtime
+
+        writer = self._resolve_writer()
+        if writer is not None:
+            await writer.start()
+
+        cache = self._cache
+        if cache is not None:
+            await cache.start()
+
+        # 初始化 Kafka consumer（启动失败不 crash，交给消费循环重连）
+        consumer = KafkaConsumerService(
+            self._kafka_config,
+            self._consumer_config,
+            topic=self.spec.topic or None,
+        )
+        try:
+            await consumer.start()
+        except Exception as e:
+            logger.error(
+                "consumer_startup_failed_degraded",
+                error=str(e),
+                bootstrap_servers=self._kafka_config.bootstrap_servers,
+            )
+
+        dlq = self._build_dlq()
+        if dlq is not None:
+            await dlq.start()
+
+        self.runtime = ConsumeRuntime(
+            spec=self.spec,
+            consumer_config=self._consumer_config,
+            writer=writer,
+            consumer=consumer,
+            codec=self._codec,
+            existence_ttl_seconds=self._existence_ttl_seconds,
+            persist_policy=self.spec.persist_policy,
+            cache=cache,
+            dlq=dlq,
+        )
+        return self.runtime
+
+    # ---- 停机 ----
+
+    def request_stop(self) -> None:
+        """请求停机（信号处理/宿主调用）。"""
+        if self.runtime is not None:
+            self.runtime.running = False
+        if self._stop_event is not None:
+            self._stop_event.set()
+
+    def _register_stop_signals(
+        self, runtime: ConsumeRuntime, stop_event: asyncio.Event
+    ) -> None:
+        """注册停机信号处理（优雅停机）。"""
+
+        def _signal_handler() -> None:
+            logger.info("shutdown_signal_received")
+            runtime.running = False
+            stop_event.set()
+
+        loop = asyncio.get_running_loop()
+        for sig in (signal.SIGTERM, signal.SIGINT):
+            try:
+                loop.add_signal_handler(sig, _signal_handler)
+            except NotImplementedError:
+                # Windows 不支持 add_signal_handler，用 signal.signal 兜底
+                signal.signal(sig, lambda s, f: _signal_handler())
+
+    # ---- 运行 ----
+
+    async def run(self) -> None:
+        """独立进程入口：组件装配 + 信号处理 + 消费循环（阻塞至停机）。
+
+        健康 HTTP 等呈现不在此装配：宿主并行启动自己的暴露实现，
+        并在 run() 返回后自行关闭。
+        """
+        runtime = await self.prepare()
+
+        stop_event = asyncio.Event()
+        self._stop_event = stop_event
+        self._register_stop_signals(runtime, stop_event)
+
+        consume_task = asyncio.create_task(consume_loop(runtime))
+
+        # 等待停机信号
+        await stop_event.wait()
+
+        # 等待消费循环退出（最多 30 秒）
+        try:
+            await asyncio.wait_for(consume_task, timeout=30.0)
+        except asyncio.TimeoutError:
+            logger.warning("shutdown_timeout_consume_loop_did_not_finish")
+
+        await self.shutdown()
+        logger.info("app_stopped")
+
+    async def shutdown(self) -> None:
+        """关闭资源（幂等）。"""
+        runtime = self.runtime
+        if runtime is None:
+            return
+        if runtime.consumer is not None:
+            await runtime.consumer.stop()
+        if runtime.dlq is not None:
+            await runtime.dlq.stop()
+        if runtime.cache is not None:
+            await runtime.cache.close()
+        if runtime.writer is not None:
+            await runtime.writer.close()
+
+    # ---- 观测 ----
+
+    async def health_snapshot(self) -> ConsumerHealthResponse:
+        """健康快照（数据归框架，暴露方式归使用方）。"""
+        runtime = self.runtime
+        if runtime is None:
+            return ConsumerHealthResponse(
+                status="stopped",
+                kafka="disconnected",
+                database=None,
+                redis=None,
+                pending_count=0,
+                lag=0,
+                quarantined_count=0,
+            )
+        return await collect_consumer_health(runtime)
+
+
+__all__ = ["ConsumerWorker"]

streamgate/db/backfill.py

@@ -0,0 +1,145 @@
+"""回源实现：NoBackfill / SqlBackfill（BackfillSource 协议的内置实现）。"""
+
+import asyncio
+from datetime import datetime
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from streamgate.obs.logging import logger
+from streamgate.protocols import JsonObject
+
+
+def _jsonable(value: object) -> object:
+    """datetime 实例/字符串统一为 ISO（与上游摘要模型序列化行为对齐：
+    原实现经 pydantic 包装，DB 驱动返回的 "YYYY-MM-DD HH:MM:SS" 字符串
+    会被解析后以 ISO 重新序列化；此处保持同一缓存格式）。"""
+    if isinstance(value, datetime):
+        return value.isoformat()
+    if (
+        isinstance(value, str)
+        and len(value) >= 19
+        and value[4] == "-"
+        and value[7] == "-"
+    ):
+        try:
+            return datetime.fromisoformat(value).isoformat()
+        except ValueError:
+            return value
+    return value
+
+
+class SqlBackfill:
+    """SQL 回源：整实体加载 slot→summary，同 slot 多行取"最新"一行。
+
+    order_columns 按优先级降序排列（如 ["tested_at", "seq_no"]），None 视为最小；
+    summary_columns: summary 键 → 列名（值为 None 表示输出固定 null）。
+    调用层超时（query_wait_seconds），超时抛 asyncio.TimeoutError。
+    """
+
+    def __init__(
+        self,
+        engine: AsyncEngine,
+        *,
+        table: str,
+        entity_column: str,
+        slot_column: str,
+        summary_columns: dict[str, str | None],
+        order_columns: list[str],
+        query_wait_seconds: float = 8.0,
+        component_label: str = "sql_backfill",
+    ) -> None:
+        self._engine: AsyncEngine | None = engine
+        self._table = table
+        self._entity_column = entity_column
+        self._slot_column = slot_column
+        self._summary_columns = summary_columns
+        self._order_columns = list(order_columns)
+        self._query_wait_seconds = query_wait_seconds
+        self._label = component_label
+        # 行布局固定：slot, *order_columns, *extra_summary_columns
+        # （summary_columns 值为 None 的键不在 SELECT 中，恒输出 null）
+        self._extra_summary: list[tuple[str, str]] = [
+            (key, col) for key, col in summary_columns.items() if col is not None
+        ]
+        self._null_summary_keys = [
+            key for key, col in summary_columns.items() if col is None
+        ]
+        select_cols = [slot_column, *order_columns, *(col for _, col in self._extra_summary)]
+        # 列名/表名来自使用方受控配置，非用户输入，无注入面
+        self._sql = text(
+            f"SELECT {', '.join(select_cols)} "
+            f"FROM {table} WHERE {entity_column} = :e"
+        )
+        logger.info(
+            f"{component_label}_initialized",
+            engine="sqlite" if "sqlite" in str(getattr(engine, "url", "")).lower() else "mssql",
+        )
+
+    async def close(self) -> None:
+        if self._engine is None:
+            return
+        await self._engine.dispose()
+        self._engine = None
+        logger.info(f"{self._label}_closed")
+
+    async def check_health(self) -> bool:
+        ok, _ = await self.check_health_detail()
+        return ok
+
+    async def check_health_detail(self) -> tuple[bool, str | None]:
+        """连接探测：返回 (是否可用, 错误信息)；失败时错误信息供调用方 ERROR 日志。"""
+        if self._engine is None:
+            return False, "engine not initialized"
+        try:
+            async with self._engine.connect() as conn:
+                await conn.execute(text("SELECT 1"))
+            return True, None
+        except Exception as e:
+            logger.debug(f"{self._label}_health_failed", error=str(e))
+            return False, str(e)
+
+    async def load(self, entity: str) -> dict[str, JsonObject]:
+        if self._engine is None:
+            raise RuntimeError("SqlBackfill not started, engine not injected")
+        try:
+            rows = await asyncio.wait_for(
+                self._fetch_rows(entity), timeout=self._query_wait_seconds
+            )
+        except asyncio.TimeoutError:
+            logger.error(
+                f"{self._label}_load_timeout",
+                entity=entity,
+                wait_seconds=self._query_wait_seconds,
+            )
+            raise
+        return self._merge_rows(rows)
+
+    async def _fetch_rows(self, entity: str) -> list[tuple[object, ...]]:
+        if self._engine is None:
+            raise RuntimeError("SqlBackfill not started, engine not injected")
+        async with self._engine.connect() as conn:
+            result = await conn.execute(self._sql, {"e": entity})
+            return [tuple(row) for row in result.fetchall()]
+
+    def _merge_rows(self, rows: list[tuple[object, ...]]) -> dict[str, JsonObject]:
+        """存量重复行：每组 slot 取排序列最大的一行（幂等摘要语义）。
+
+        row 布局与 SELECT 列序一致：slot, *order_columns, *extra_summary_columns。
+        """
+        n_order = len(self._order_columns)
+        best: dict[str, tuple[tuple[object, ...], JsonObject]] = {}
+        for row in rows:
+            slot = row[0]
+            if not isinstance(slot, str):
+                continue  # 历史脏行防御（None/非字符串主键），不应出现
+            summary: JsonObject = {}
+            for key in self._null_summary_keys:
+                summary[key] = None
+            for i, (key, _col) in enumerate(self._extra_summary):
+                summary[key] = _jsonable(row[1 + n_order + i])
+            sort_values = row[1 : 1 + n_order]
+            sort_key = tuple((v is not None, v if v is not None else 0) for v in sort_values)
+            if slot not in best or sort_key > best[slot][0]:
+                best[slot] = (sort_key, summary)
+        return {slot: meta for slot, (_, meta) in best.items()}

streamgate/db/dialects/mssql.py

@@ -0,0 +1,126 @@
+"""MSSQL 方言：多行 VALUES + MERGE 幂等 upsert（按参数预算分块）。
+
+SQLAlchemy 2.0 的 mssql 方言未提供 MERGE 构造，用 text() 写原生 SQL。
+
+并发前提（生产实测教训，勿"简化"）：
+- HOLDLOCK 是并发正确性的必需项（序列化同键 MERGE，防止双方同时
+  判定 NOT MATCHED 而双插），不得移除；
+- 外部写入方竞态可能在存量表中制造同键重复行，MERGE"target N 行匹配
+  全部 UPDATE"语义是对该现实的对冲，不得收紧为单行更新；
+- 阻塞/死锁（1205 被选为 victim）由整事务超时与上层通用重试兜底，
+  事务回滚天然保原子性。
+"""
+
+from collections.abc import Iterator
+
+from sqlalchemy import String, TextClause, text
+from sqlalchemy import inspect as sa_inspect
+from sqlalchemy.dialects import mssql
+from sqlmodel import SQLModel
+
+from streamgate.protocols import JsonObject
+
+# SQL Server 单语句参数上限约 2100，预留 sp_executesql 内部参数开销后的
+# 安全预算；每块行数 = 预算 // 字段数，随字段数动态计算（不写死行数）。
+SAFE_PARAM_BUDGET = 2000
+
+
+def mssql_cast_types(model: type[SQLModel]) -> dict[str, str]:
+    """按表元数据生成各属性的 MSSQL CAST 目标类型（键为 Python 属性名）。
+
+    应对 B4：VALUES 行构造器中某列全为 NULL 参数时，类型推导可能给出与
+    目标列不匹配的类型，显式 CAST 固定每列类型。字符串列统一 NVARCHAR(max)：
+    驱动（aioodbc/pyodbc）本就按 Unicode 声明字符串参数，与既有单行 MERGE
+    的比较语义一致，且规避 CAST 到 VARCHAR 时按库默认排序规则代码页转换
+    的字符损失风险；其余类型按 MSSQL 方言编译列元数据（FLOAT / DATETIME）。
+    """
+    dialect = mssql.dialect()
+    types: dict[str, str] = {}
+    for prop in sa_inspect(model).column_attrs:
+        col_type = prop.columns[0].type
+        # SQLModel 字符串列是 TypeDecorator(AutoString)，需下钻 impl 判定字符串族
+        impl = getattr(col_type, "impl", col_type)
+        if isinstance(impl, type):
+            is_string = issubclass(impl, String)
+        else:
+            is_string = isinstance(impl, String)
+        types[prop.key] = "NVARCHAR(max)" if is_string else col_type.compile(dialect)
+    return types
+
+
+def chunk_by_param_budget(
+    rows: list[JsonObject],
+    field_count: int,
+) -> Iterator[list[JsonObject]]:
+    """按参数预算分块：每块行数 = SAFE_PARAM_BUDGET // 字段数。"""
+    max_rows = max(1, SAFE_PARAM_BUDGET // field_count)
+    for i in range(0, len(rows), max_rows):
+        yield rows[i : i + max_rows]
+
+
+def dedup_rows(
+    rows: list[JsonObject],
+    keys: list[str],
+) -> list[JsonObject]:
+    """按 upsert 键去重，保留最后一条。
+
+    防止多行 VALUES + MERGE 因 source 内部重复（M 行 source 匹配同一行
+    target，MERGE 报 "attempted to UPDATE or DELETE the same row more
+    than once"）导致整批失败成为毒批。业务上一键一条，正常不会重复；
+    此为防御性去重。
+    """
+    seen: dict[tuple[object, ...], JsonObject] = {}
+    for d in rows:
+        key = tuple(d.get(k) for k in keys)
+        seen[key] = d  # 后者覆盖前者
+    return list(seen.values())
+
+
+def build_multirow_merge(
+    table_name: str,
+    fields: list[str],
+    keys: list[str],
+    column_names: dict[str, str],
+    cast_types: dict[str, str],
+    rows: list[JsonObject],
+) -> tuple[TextClause, JsonObject]:
+    """构建多行 VALUES + MERGE 语句及参数（分块后的一块）。
+
+    - 参数命名 r{行号}_{属性名}，每行每列唯一；字段名来自模型元数据，无注入面；
+    - 列名经 column_names 映射（属性名 → DB 列名，保留原始列名的大小写与特殊字符）；
+    - ON / UPDATE / INSERT 子句与生产验证过的单行 MERGE 模板逐字一致。
+    """
+    def _col(field: str) -> str:
+        return column_names.get(field, field)
+
+    values_rows: list[str] = []
+    params: JsonObject = {}
+    for i, row in enumerate(rows):
+        casts: list[str] = []
+        for field in fields:
+            param = f"r{i}_{field}"
+            params[param] = row.get(field)
+            casts.append(f"CAST(:{param} AS {cast_types[field]})")
+        values_rows.append(f"({', '.join(casts)})")
+
+    source_rows = ",\n            ".join(values_rows)
+    source_aliases = ", ".join(f"[{_col(f)}]" for f in fields)
+    update_fields = [f for f in fields if f not in keys]
+    set_clause = ", ".join(f"t.[{_col(f)}] = s.[{_col(f)}]" for f in update_fields)
+    insert_col_list = ", ".join(f"[{_col(f)}]" for f in fields)
+    insert_val_list = ", ".join(f"s.[{_col(f)}]" for f in fields)
+    on_clause = " AND ".join(f"t.[{_col(k)}] = s.[{_col(k)}]" for k in keys)
+
+    sql = text(
+        f"""
+        MERGE INTO {table_name} WITH (HOLDLOCK) AS t
+        USING (VALUES
+            {source_rows}
+        ) AS s({source_aliases})
+        ON {on_clause}
+        WHEN MATCHED THEN UPDATE SET {set_clause}
+        WHEN NOT MATCHED THEN INSERT ({insert_col_list})
+        VALUES ({insert_val_list});
+        """
+    )
+    return sql, params

streamgate/db/dialects/sqlite.py

@@ -0,0 +1,57 @@
+"""SQLite 方言（dev/测试路径）：ON CONFLICT 优先，无唯一约束时 DELETE+INSERT 兜底。"""
+
+from sqlalchemy import text
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlmodel import SQLModel
+
+from streamgate.protocols import JsonObject
+
+
+def _constraint_covers_keys(model: type[SQLModel], keys: list[str]) -> bool:
+    """表上是否存在恰好覆盖 upsert 键的主键/唯一约束（ON CONFLICT 前提）。"""
+    table = model.__table__  # type: ignore[reportAttributeAccessIssue]  # SQLModel stub lacks SQLAlchemy Table
+    key_set = set(keys)
+    for constraint in (*table.constraints, *table.indexes):
+        cols = {c.name for c in getattr(constraint, "columns", [])}
+        if cols and cols == key_set and getattr(constraint, "unique", False):
+            return True
+    return False
+
+
+async def upsert_on_conflict(
+    session: AsyncSession,
+    model: type[SQLModel],
+    rows: list[JsonObject],
+    keys: list[str],
+    fields: list[str],
+) -> None:
+    """INSERT ... ON CONFLICT(keys) DO UPDATE（要求键上有唯一约束/主键）。"""
+    stmt = sqlite_insert(model).values(rows)
+    update_cols = {
+        k: stmt.excluded[k] for k in fields if k not in keys
+    }
+    stmt = stmt.on_conflict_do_update(index_elements=keys, set_=update_cols)
+    await session.execute(stmt)
+
+
+async def delete_insert_by_keys(
+    session: AsyncSession,
+    model: type[SQLModel],
+    rows: list[JsonObject],
+    keys: list[str],
+) -> None:
+    """无唯一约束时的幂等兜底：事务内按键 DELETE 后 INSERT（仅测试环境用）。"""
+    pairs = [tuple(d.get(k) for k in keys) for d in rows]
+    pairs = [p for p in pairs if all(v is not None for v in p)]
+    table_name = model.__tablename__
+    where = " AND ".join(f"{k} = :k{i}" for i, k in enumerate(keys))
+    stmt = text(f"DELETE FROM {table_name} WHERE {where}")
+    for pair in pairs:
+        params = {f"k{i}": v for i, v in enumerate(pair)}
+        await session.execute(stmt, params)
+    await session.execute(model.__table__.insert(), rows)  # type: ignore[reportAttributeAccessIssue]  # SQLModel stub lacks SQLAlchemy Table
+
+
+def supports_on_conflict(model: type[SQLModel], keys: list[str]) -> bool:
+    return _constraint_covers_keys(model, keys)

streamgate/db/engines.py

@@ -0,0 +1,113 @@
+"""引擎工厂（读写池、超时钩子，唯一建引擎点）。"""
+
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import (
+    AsyncEngine,
+    AsyncSession,
+    async_sessionmaker,
+    create_async_engine,
+)
+
+from streamgate.config import DbConfig
+from streamgate.obs.logging import logger
+
+
+def make_query_timeout_hook(timeout_seconds: int):
+    """SQLAlchemy connect 事件钩子：建连后设置 pyodbc 语句级查询超时（秒）。
+
+    生产驱动（pyodbc 系 / aioodbc）通过 dbapi_conn.timeout 控制，
+    超时后连接真正释放；SQLite 无此概念（dev 不适用）。
+    aioodbc 异步引擎的 connect 事件拿到的是 AsyncAdapt 包装连接，
+    timeout 需落在底层 driver_connection（aioodbc/pyodbc 真实连接）上，
+    故优先取 driver_connection 再 setattr。
+    aioodbc 0.5.x 的 Connection.timeout 是只读 property（无 setter），
+    setattr 会抛 AttributeError；真实 pyodbc 连接存于 Connection._conn，
+    SQLAlchemy 的 AsyncAdapt autocommit setter 亦穿透 _conn 设置（模式一致），
+    故失败时穿透 _conn 再设一次。
+    驱动差异防御：属性不存在/设置失败仅 WARN，不阻断建连；
+    调用层 wait_for 仍是硬保障。
+    """
+
+    def _on_connect(dbapi_conn: object, _record: object) -> None:
+        try:
+            driver = getattr(dbapi_conn, "driver_connection", dbapi_conn)
+            _set_driver_timeout(driver, timeout_seconds)
+        except Exception as e:
+            logger.warning("db_query_timeout_set_failed", error=str(e))
+
+    return _on_connect
+
+
+def _set_driver_timeout(conn: object, timeout_seconds: int) -> None:
+    try:
+        setattr(conn, "timeout", timeout_seconds)
+    except AttributeError:
+        underlying = getattr(conn, "_conn", None)
+        if underlying is None:
+            raise
+        setattr(underlying, "timeout", timeout_seconds)
+
+
+def _is_sqlite(config: DbConfig) -> bool:
+    return "sqlite" in config.require_connection_string().lower()
+
+
+def _create_engine(config: DbConfig, pool_size: int, max_overflow: int, timeout_seconds: int) -> AsyncEngine:
+    """引擎创建单点：内置 sqlite/mssql 分支、池参数、timeout 钩子挂载。
+
+    C7 装配校验：connection_string 缺失即在此抛错（含修复指引），
+    对齐 KAFKA__TOPIC / CONSUMER__GROUP_ID 的"必填即启动失败"语义。
+    """
+    connection_string = config.require_connection_string()
+    if _is_sqlite(config):
+        return create_async_engine(
+            connection_string,
+            echo=config.echo,
+            connect_args={"check_same_thread": False},
+        )
+    engine = create_async_engine(
+        connection_string,
+        echo=config.echo,
+        pool_size=pool_size,
+        max_overflow=max_overflow,
+        pool_timeout=config.pool_timeout_seconds,  # 池耗尽快速失败
+        pool_recycle=3600,
+        pool_pre_ping=True,
+    )
+    # 建连即设驱动语句超时（pyodbc 系 dbapi_conn.timeout），
+    # 钩子内含驱动差异防御，失败仅 WARN 不阻断建连。
+    if timeout_seconds > 0:
+        event.listen(
+            engine.sync_engine,
+            "connect",
+            make_query_timeout_hook(timeout_seconds),
+        )
+    return engine
+
+
+def create_write_engine(config: DbConfig) -> AsyncEngine:
+    """consumer 写库引擎（池参数见 DB__WRITE_POOL_* 配置键）。"""
+    return _create_engine(
+        config,
+        pool_size=config.write_pool_size,
+        max_overflow=config.write_pool_max_overflow,
+        timeout_seconds=config.write_timeout_seconds,
+    )
+
+
+def create_read_engine(config: DbConfig) -> AsyncEngine:
+    """ingest 读库引擎（冷实体回源专用）。"""
+    return _create_engine(
+        config,
+        pool_size=config.read_pool_size,
+        max_overflow=config.read_pool_max_overflow,
+        timeout_seconds=config.query_timeout_seconds,
+    )
+
+
+def async_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
+    """统一 session factory 参数（expire_on_commit=False）。"""
+    return async_sessionmaker(
+        engine,
+        expire_on_commit=False,
+    )