streamgate 0.3.0__tar.gz → 0.3.1__tar.gz

{streamgate-0.3.0 → streamgate-0.3.1}/.github/workflows/ci.yml

@@ -83,8 +83,9 @@ jobs:
           uv venv bare
           uv pip install --python bare/bin/python .
       - name: "Smoke: dependency tree is pure (no DB/redis/httpx carriers)"
+        # uv venv 不内置 pip：用 uv pip list 读取裸环境依赖清单
         run: |
-          bare/bin/python -m pip list --format=freeze > deps.txt
+          uv pip list --python bare/bin/python --format=freeze > deps.txt
           cat deps.txt
           for pkg in sqlalchemy sqlmodel redis aiosqlite aioodbc httpx; do
             if grep -qi "^${pkg}==" deps.txt; then

streamgate-0.3.1/.import_linter_cache/a1215eee6d5c7b9f11de3767d6965c4a9c703b3e.data.json

@@ -0,0 +1 @@
+{"imports_by_module":{"streamgate":[["streamgate.ingest.admission.no_admission",false,38,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.transport.codec",false,74,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.config",false,25,"from streamgate.config import BackpressureConfig, ConsumerConfig, KafkaConfig"],["streamgate.resilience.backpressure",false,66,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.specs",false,73,"from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT"],["streamgate.ingest.producer",false,40,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.obs.logging",false,41,"from streamgate.obs.logging import configure_logger, logger"],["streamgate.consumer.runner",false,36,"from streamgate.consumer.runner import ConsumerWorker"],["streamgate.ingest.gateway",false,39,"from streamgate.ingest.gateway import IngestGateway"],["streamgate.resilience.health",false,67,"from streamgate.resilience.health import ("],["streamgate.consumer.dlq",false,27,"from streamgate.consumer.dlq import ("],["streamgate.consumer.loop",false,35,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.transport.kafka",false,75,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.obs.metrics",false,42,"from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink"],["streamgate.protocols",false,43,"from streamgate.protocols import ("],["streamgate.ingest.admission.in_memory",false,37,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.consumer.classifier",false,26,"from streamgate.consumer.classifier import DefaultErrorClassifier"]],"streamgate.config":[["pydantic",false,8,"from pydantic import BaseModel, field_validator"]],"streamgate.consumer":[],"streamgate.consumer.classifier":[["streamgate.protocols",false,20,"from streamgate.protocols import ErrorKind"],["asyncio",false,16,"import asyncio"],["aiokafka",false,18,"from aiokafka.errors import KafkaError"]],"streamgate.consumer.dlq":[["aiokafka",false,20,"from aiokafka import AIOKafkaProducer"],["streamgate.obs.logging",false,23,"from streamgate.obs.logging import logger"],["datetime",false,18,"from datetime import datetime, timezone"],["streamgate.protocols",false,24,"from streamgate.protocols import ErrorKind, JsonObject, RecordWriter"],["dataclasses",false,17,"from dataclasses import dataclass, field"],["asyncio",false,15,"import asyncio"],["streamgate.config",false,22,"from streamgate.config import KafkaConfig"],["json",false,16,"import json"]],"streamgate.consumer.loop":[["streamgate.transport.kafka",false,37,"from streamgate.transport.kafka import KafkaConsumerService, KafkaRecord"],["streamgate.specs",false,36,"from streamgate.specs import ConsumeSpec"],["streamgate.obs.metrics",false,25,"from streamgate.obs.metrics import DEFAULT_METRICS, MetricsSink"],["asyncio",false,5,"import asyncio"],["streamgate.consumer.dlq",false,15,"from streamgate.consumer.dlq import ("],["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.consumer.classifier",false,14,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig"],["collections",false,7,"from collections.abc import Callable"],["aiokafka",false,11,"from aiokafka.structs import OffsetAndMetadata, TopicPartition"],["aiokafka",false,10,"from aiokafka.errors import CommitFailedError"],["streamgate.protocols",false,26,"from streamgate.protocols import ("],["time",false,6,"import time"],["streamgate.obs.logging",false,24,"from streamgate.obs.logging import logger"]],"streamgate.consumer.runner":[["streamgate.specs",false,22,"from streamgate.specs import ConsumeSpec"],["streamgate.consumer.dlq",false,14,"from streamgate.consumer.dlq import DlqProducer"],["collections",false,11,"from collections.abc import Awaitable, Callable"],["signal",false,10,"import signal"],["streamgate.obs.logging",false,16,"from streamgate.obs.logging import logger"],["streamgate.transport.kafka",false,24,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.transport.codec",false,23,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.resilience.health",false,18,"from streamgate.resilience.health import ("],["streamgate.protocols",false,17,"from streamgate.protocols import ErrorClassifier, MessageCodec"],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig, KafkaConfig"],["streamgate.consumer.loop",false,15,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["asyncio",false,9,"import asyncio"]],"streamgate.contrib":[],"streamgate.contrib._deps":[["typing",false,3,"from typing import NoReturn"]],"streamgate.contrib.http_probe":[["streamgate.contrib.http_probe.probe_signal",false,12,"from streamgate.contrib.http_probe.probe_signal import ("],["streamgate.contrib._deps",false,9,"from streamgate.contrib._deps import require_extra_import"]],"streamgate.contrib.http_probe.probe_signal":[["streamgate.protocols",false,25,"from streamgate.protocols import BackpressureSnapshot, ProbeResult"],["httpx",false,22,"import httpx"],["time",false,20,"import time"],["asyncio",false,19,"import asyncio"],["streamgate",false,24,"from streamgate import BackpressureConfig, logger"]],"streamgate.contrib.mssql_sink":[["streamgate.contrib.sql_sink",false,10,"from streamgate.contrib.sql_sink import ("]],"streamgate.contrib.redis_admission":[["streamgate.contrib.redis_admission.admission",false,11,"from streamgate.contrib.redis_admission.admission import ("],["streamgate.contrib.redis_admission.existence",false,21,"from streamgate.contrib.redis_admission.existence import ("],["streamgate.contrib.redis_admission.config",false,20,"from streamgate.contrib.redis_admission.config import RedisConfig"],["streamgate.contrib._deps",false,8,"from streamgate.contrib._deps import require_extra_import"]],"streamgate.contrib.redis_admission.admission":[["dataclasses",false,12,"from dataclasses import dataclass"],["redis",false,16,"from redis.exceptions import RedisError"],["streamgate.protocols",false,21,"from streamgate.protocols import ("],["streamgate.contrib.redis_admission.config",false,19,"from streamgate.contrib.redis_admission.config import RedisConfig"],["collections",false,11,"from collections.abc import Callable"],["streamgate",false,18,"from streamgate import logger"],["enum",false,13,"from enum import Enum"],["typing",false,14,"from typing import Generic"],["streamgate.contrib.redis_admission.existence",false,20,"from streamgate.contrib.redis_admission.existence import RedisExistenceCache"],["asyncio",false,10,"import asyncio"]],"streamgate.contrib.redis_admission.config":[["pydantic",false,6,"from pydantic import BaseModel"]],"streamgate.contrib.redis_admission.existence":[["typing",false,12,"from typing import TypeVar"],["collections",false,11,"from collections.abc import Awaitable"],["asyncio",false,9,"import asyncio"],["json",false,10,"import json"],["redis",false,14,"import redis.asyncio as aioredis"],["streamgate.contrib.redis_admission.config",false,17,"from streamgate.contrib.redis_admission.config import RedisConfig"],["streamgate",false,16,"from streamgate import logger"]],"streamgate.contrib.sql_sink":[["streamgate.contrib.sql_sink.engines",false,15,"from streamgate.contrib.sql_sink.engines import ("],["streamgate.contrib.sql_sink.upsert",false,19,"from streamgate.contrib.sql_sink.upsert import ("],["streamgate.contrib.sql_sink.classifier",false,13,"from streamgate.contrib.sql_sink.classifier import SQLAlchemyErrorClassifier"],["streamgate.contrib._deps",false,9,"from streamgate.contrib._deps import require_extra_import"],["streamgate.contrib.sql_sink.backfill",false,12,"from streamgate.contrib.sql_sink.backfill import SqlBackfill"],["streamgate.contrib.sql_sink.config",false,14,"from streamgate.contrib.sql_sink.config import DbConfig"]],"streamgate.contrib.sql_sink._dialects":[],"streamgate.contrib.sql_sink._dialects.mssql":[["sqlalchemy",false,17,"from sqlalchemy import inspect as sa_inspect"],["collections",false,14,"from collections.abc import Iterator"],["sqlmodel",false,19,"from sqlmodel import SQLModel"],["sqlalchemy",false,16,"from sqlalchemy import String, TextClause, text"],["sqlalchemy",false,18,"from sqlalchemy.dialects import mssql"],["streamgate.protocols",false,21,"from streamgate.protocols import JsonObject"]],"streamgate.contrib.sql_sink._dialects.sqlite":[["sqlalchemy",false,5,"from sqlalchemy.ext.asyncio import AsyncSession"],["sqlmodel",false,6,"from sqlmodel import SQLModel"],["sqlalchemy",false,4,"from sqlalchemy.dialects.sqlite import insert as sqlite_insert"],["streamgate.protocols",false,8,"from streamgate.protocols import JsonObject"],["sqlalchemy",false,3,"from sqlalchemy import text"]],"streamgate.contrib.sql_sink.backfill":[["sqlalchemy",false,10,"from sqlalchemy import text"],["sqlalchemy",false,11,"from sqlalchemy.ext.asyncio import AsyncEngine"],["asyncio",false,7,"import asyncio"],["streamgate",false,13,"from streamgate import logger"],["datetime",false,8,"from datetime import datetime"],["streamgate.protocols",false,14,"from streamgate.protocols import JsonObject"]],"streamgate.contrib.sql_sink.classifier":[["streamgate",false,32,"from streamgate import ErrorKind"],["sqlalchemy",false,23,"from sqlalchemy.exc import ("],["asyncio",false,18,"import asyncio"],["collections",false,20,"from collections.abc import Iterator"],["sqlalchemy",false,28,"from sqlalchemy.exc import ("],["aiokafka",false,22,"from aiokafka.errors import KafkaError"],["re",false,19,"import re"]],"streamgate.contrib.sql_sink.config":[["pydantic",false,7,"from pydantic import BaseModel"]],"streamgate.contrib.sql_sink.engines":[["sqlalchemy",false,4,"from sqlalchemy.ext.asyncio import ("],["streamgate.contrib.sql_sink.config",false,12,"from streamgate.contrib.sql_sink.config import DbConfig"],["sqlalchemy",false,3,"from sqlalchemy import event"],["streamgate",false,11,"from streamgate import logger"]],"streamgate.contrib.sql_sink.upsert":[["asyncio",false,8,"import asyncio"],["streamgate.contrib.sql_sink.config",false,20,"from streamgate.contrib.sql_sink.config import DbConfig"],["sqlalchemy",false,12,"from sqlalchemy import text"],["streamgate",false,16,"from streamgate import logger"],["streamgate.protocols",false,21,"from streamgate.protocols import JsonObject, WriteResult"],["streamgate.contrib.sql_sink._dialects.mssql",false,17,"from streamgate.contrib.sql_sink._dialects import mssql as mssql_dialect"],["streamgate.contrib.sql_sink._dialects.mssql",false,19,"from streamgate.contrib.sql_sink._dialects.mssql import mssql_cast_types"],["sqlalchemy",false,11,"from sqlalchemy import inspect as sa_inspect"],["streamgate.contrib.sql_sink._dialects.sqlite",false,18,"from streamgate.contrib.sql_sink._dialects import sqlite as sqlite_dialect"],["collections",false,9,"from collections.abc import Callable"],["sqlmodel",false,14,"from sqlmodel import SQLModel"],["sqlalchemy",false,13,"from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker"]],"streamgate.contrib.sqlite_sink":[["streamgate.contrib.sql_sink",false,9,"from streamgate.contrib.sql_sink import ("]],"streamgate.ingest":[],"streamgate.ingest.admission":[],"streamgate.ingest.admission.in_memory":[["typing",false,13,"from typing import Generic"],["streamgate.protocols",false,15,"from streamgate.protocols import Decision, JsonObject, RecordT"],["collections",false,12,"from collections.abc import Callable"]],"streamgate.ingest.admission.no_admission":[["streamgate.protocols",false,5,"from streamgate.protocols import Decision, RecordT"],["typing",false,3,"from typing import Generic"]],"streamgate.ingest.gateway":[["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.resilience.health",false,26,"from streamgate.resilience.health import ("],["streamgate.ingest.producer",false,14,"from streamgate.ingest.producer import KafkaProducerService"],["typing",false,9,"from typing import Generic"],["streamgate.specs",false,30,"from streamgate.specs import IngestBinding, IngestRecordT"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"],["streamgate.ingest.admission.no_admission",false,13,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.config",false,11,"from streamgate.config import BackpressureConfig, KafkaConfig"],["streamgate.transport.codec",false,31,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.ingest.admission.in_memory",false,12,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.resilience.backpressure",false,25,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.protocols",false,16,"from streamgate.protocols import ("]],"streamgate.ingest.producer":[["aiokafka",false,14,"from aiokafka.errors import KafkaConnectionError, KafkaError"],["time",false,10,"import time"],["streamgate.config",false,16,"from streamgate.config import KafkaConfig"],["asyncio",false,8,"import asyncio"],["json",false,9,"import json"],["aiokafka",false,13,"from aiokafka import AIOKafkaProducer"],["streamgate.protocols",false,18,"from streamgate.protocols import JsonObject"],["streamgate.obs.logging",false,17,"from streamgate.obs.logging import logger"],["datetime",false,11,"from datetime import datetime, timezone"]],"streamgate.obs":[],"streamgate.obs.logging":[["loguru",false,6,"from loguru._handler import Message"],["sys",false,2,"import sys"],["json",false,1,"import json"],["datetime",false,3,"from datetime import timezone"],["loguru",false,5,"from loguru import logger as _logger"]],"streamgate.obs.metrics":[["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"],["typing",false,8,"from typing import Protocol"]],"streamgate.protocols":[["dataclasses",false,8,"from dataclasses import dataclass, field"],["enum",false,10,"from enum import Enum"],["datetime",false,9,"from datetime import datetime"],["typing",false,11,"from typing import Protocol, TypeVar, runtime_checkable"],["collections",false,7,"from collections.abc import Awaitable, Callable"]],"streamgate.resilience":[],"streamgate.resilience.backpressure":[["streamgate.protocols",false,10,"from streamgate.protocols import AllowAllSignal, BackpressureSnapshot"]],"streamgate.resilience.health":[["asyncio",false,7,"import asyncio"],["collections",false,9,"from collections.abc import Awaitable"],["pydantic",false,13,"from pydantic import BaseModel, Field"],["time",false,8,"import time"],["typing",false,11,"from typing import Protocol"],["datetime",false,10,"from datetime import datetime, timezone"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"]],"streamgate.specs":[["typing",false,12,"from typing import Generic, Literal, TypeVar"],["collections",false,10,"from collections.abc import Callable, Hashable"],["pydantic",false,14,"from pydantic import BaseModel"],["dataclasses",false,11,"from dataclasses import dataclass, field"],["streamgate.protocols",false,16,"from streamgate.protocols import ("]],"streamgate.transport":[],"streamgate.transport.codec":[["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"],["json",false,8,"import json"],["streamgate.protocols",false,11,"from streamgate.protocols import Envelope, JsonObject"]],"streamgate.transport.kafka":[["streamgate.config",false,10,"from streamgate.config import ConsumerConfig, KafkaConfig"],["aiokafka",false,7,"from aiokafka.errors import CommitFailedError, KafkaConnectionError, KafkaError"],["streamgate.obs.logging",false,11,"from streamgate.obs.logging import logger"],["aiokafka",false,8,"from aiokafka.structs import ConsumerRecord, OffsetAndMetadata, TopicPartition"],["aiokafka",false,6,"from aiokafka.consumer.group_coordinator import GroupCoordinator"],["asyncio",false,3,"import asyncio"],["aiokafka",false,5,"from aiokafka import AIOKafkaConsumer"]]},"version":2}

streamgate-0.3.1/.import_linter_cache/streamgate.meta.json

@@ -0,0 +1 @@
+{"streamgate.contrib.sql_sink": 1788836043.336221, "streamgate.contrib.http_probe.probe_signal": 1788836090.470792, "streamgate.obs": 1788742270.5761602, "streamgate.transport": 1788742270.5761602, "streamgate.contrib._deps": 1788835802.9044309, "streamgate.contrib.redis_admission.existence": 1788835837.2841742, "streamgate.contrib": 1788835802.905441, "streamgate.ingest.admission.no_admission": 1788759910.7492158, "streamgate.protocols": 1788780004.1189399, "streamgate.contrib.mssql_sink": 1788836174.9297483, "streamgate.consumer.loop": 1788778765.328234, "streamgate.contrib.redis_admission.admission": 1788835898.0445402, "streamgate.consumer.dlq": 1788778795.1676013, "streamgate.ingest.admission": 1788742270.5761602, "streamgate.contrib.sql_sink.upsert": 1788836025.5259016, "streamgate.contrib.sqlite_sink": 1788836174.9292364, "streamgate.config": 1788778673.809221, "streamgate": 1788836110.1479475, "streamgate.contrib.sql_sink._dialects.mssql": 1788836025.5239024, "streamgate.transport.kafka": 1788776206.140225, "streamgate.transport.codec": 1788749619.0487723, "streamgate.contrib.redis_admission.config": 1788835837.2772365, "streamgate.contrib.sql_sink._dialects.sqlite": 1788835977.7374504, "streamgate.ingest": 1788742270.5761602, "streamgate.ingest.admission.in_memory": 1788778617.9172873, "streamgate.contrib.http_probe": 1788836174.9308388, "streamgate.obs.metrics": 1788749264.9746833, "streamgate.contrib.sql_sink.classifier": 1788835977.7154298, "streamgate.resilience.health": 1788767769.120218, "streamgate.resilience": 1788742270.5761602, "streamgate.consumer.classifier": 1788778605.4753034, "streamgate.resilience.backpressure": 1788778626.931601, "streamgate.obs.logging": 1788749262.353655, "streamgate.contrib.sql_sink._dialects": 1788835977.7374504, "streamgate.ingest.gateway": 1788778851.3711455, "streamgate.ingest.producer": 1788776206.139223, "streamgate.contrib.redis_admission": 1788835898.0475388, "streamgate.consumer": 1788742270.5761602, "streamgate.contrib.sql_sink.backfill": 1788835942.310305, "streamgate.consumer.runner": 1788845971.1311324, "streamgate.contrib.sql_sink.engines": 1788835942.286183, "streamgate.contrib.sql_sink.config": 1788835924.6027458, "streamgate.specs": 1788778652.1194282}

{streamgate-0.3.0 → streamgate-0.3.1}/CHANGELOG.md

@@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.1] - 2026-09-08
+
+### Fixed
+
+- **`ConsumerWorker` now manages the `ConsumeSpec.persist_policy` lifecycle**
+  (the `AdmissionPolicy` protocol promises "the framework guarantees call
+  ordering", but the consumer entry point never honored it — unlike
+  `IngestGateway`, which calls `start()`/`close()`). `prepare()` now calls
+  `await policy.start()` before starting the sink, and `shutdown()` calls
+  `await policy.close()` before closing the writer. Previously the consumer
+  side never started the policy, so `on_persisted` authoritative refresh /
+  TTL heartbeats failed on every batch (e.g. `RedisExistenceAdmission`
+  logging `cache_refresh_failed: RedisExistenceCache not started`) and the
+  failure surfaced only as per-record WARN logs. Lifecycle calls are
+  duck-typed (defensive `getattr`) so legacy policy implementations without
+  `start()`/`close()` keep working.
+- **`ConsumerWorker(cache=...)` lifecycle is now symmetric.** The bypass
+  `cache` parameter (health-probe component) is started in `prepare()` when
+  it implements `start()`, instead of being closed at shutdown without ever
+  having been started. `RedisExistenceCache.start()`/`close()` are
+  idempotent, so sharing one cache instance between the policy and the
+  bypass parameter remains safe (double start/close is a no-op).
+  The pre-fix workaround (`await cache.start()` before `worker.run()`)
+  is no longer required.
+
 ## [0.3.0] - 2026-09-08
 
 **Contrib release.** The production-grade I/O strategies that 0.2.0 demoted to

{streamgate-0.3.0 → streamgate-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: streamgate
-Version: 0.3.0
+Version: 0.3.1
 Summary: A pure-core Kafka data pipeline framework: conditional admission, reliable delivery, pluggable sinks via typed hooks.
 Project-URL: Homepage, https://github.com/pwg-code/streamgate
 Project-URL: Repository, https://github.com/pwg-code/streamgate

{streamgate-0.3.0 → streamgate-0.3.1}/src/streamgate/__init__.py

@@ -74,7 +74,7 @@ from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT
 from streamgate.transport.codec import JsonEnvelopeCodec
 from streamgate.transport.kafka import KafkaConsumerService
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 __all__ = [
     "AdmissionPolicy",

{streamgate-0.3.0 → streamgate-0.3.1}/src/streamgate/consumer/runner.py

@@ -1,12 +1,14 @@
 """消费服务装配与生命周期（机制内核，无 HTTP 呈现）。
 
-职责：init sink（spec.sink，唯一落库路径）→ 启动 consumer/DLQ → 消费循环 →
+职责：init persist_policy（协议生命周期，框架保证调用时序）→ init sink
+（spec.sink，唯一落库路径）→ 启动 consumer/DLQ → 消费循环 →
 优雅停机（flush 缓冲、leave group、关资源）→ 健康快照。
 健康数据经 health_snapshot() 暴露；端点实现归使用方。
 """
 
 import asyncio
 import signal
+from collections.abc import Awaitable, Callable
 
 from streamgate.config import ConsumerConfig, KafkaConfig
 from streamgate.consumer.dlq import DlqProducer
@@ -25,6 +27,20 @@ from streamgate.transport.kafka import KafkaConsumerService
 DEFAULT_EXISTENCE_TTL_SECONDS = 18000
 
 
+async def _lifecycle_start(component: object) -> None:
+    """启动可选组件（鸭子类型防御：未实现 start() 的历史实现跳过）。"""
+    start: Callable[[], Awaitable[None]] | None = getattr(component, "start", None)
+    if start is not None:
+        await start()
+
+
+async def _lifecycle_close(component: object) -> None:
+    """释放可选组件（鸭子类型防御：未实现 close() 的历史实现跳过）。"""
+    close: Callable[[], Awaitable[None]] | None = getattr(component, "close", None)
+    if close is not None:
+        await close()
+
+
 class ConsumerWorker:
     def __init__(
         self,
@@ -40,7 +56,8 @@ class ConsumerWorker:
         """装配消费内核。
 
         cache：可选健康探测组件（鸭子类型：实现 check_health() 即被
-        /health 快照观测，如使用方自建的准入缓存）。
+        /health 快照观测，如使用方自建的准入缓存）。实现 start()/close()
+        时生命周期由框架托管（prepare 启动、shutdown 释放）。
         error_classifier：写侧异常分类器注入点；未注入用
         DefaultErrorClassifier（只认通用异常——接 DB 务必注入对应分类器，
         参考 examples/sqlite_sink/）。
@@ -82,6 +99,13 @@ class ConsumerWorker:
         if self.runtime is not None:
             return self.runtime
 
+        # 托管策略生命周期（AdmissionPolicy 协议契约：框架保证调用时序，
+        # 与 IngestGateway.start() 对齐；旁路注入的 cache 同步托管）
+        if self.spec.persist_policy is not None:
+            await _lifecycle_start(self.spec.persist_policy)
+        if self._cache is not None:
+            await _lifecycle_start(self._cache)
+
         writer = self.spec.sink
         if writer is not None:
             await writer.start()
@@ -183,10 +207,12 @@ class ConsumerWorker:
             await runtime.consumer.stop()
         if runtime.dlq is not None:
             await runtime.dlq.stop()
+        # 策略先于旁路 cache 释放：RedisExistenceAdmission.close() 已关闭
+        # 内部 cache 及 backfill；cache.close() 幂等，双保险亦安全
+        if runtime.persist_policy is not None:
+            await _lifecycle_close(runtime.persist_policy)
         if runtime.cache is not None:
-            close = getattr(runtime.cache, "close", None)
-            if close is not None:
-                await close()
+            await _lifecycle_close(runtime.cache)
         if runtime.writer is not None:
             await runtime.writer.close()
 

{streamgate-0.3.0 → streamgate-0.3.1}/src/streamgate/contrib/redis_admission/existence.py

@@ -9,7 +9,7 @@
 import asyncio
 import json
 from collections.abc import Awaitable
-from typing import TypeVar
+from typing import TypeVar, cast
 
 import redis.asyncio as aioredis
 
@@ -145,7 +145,14 @@ class RedisExistenceCache:
     # ---------- 读 ----------
 
     async def get_field_meta(self, entity: str, slot: str) -> dict[str, object] | None:
-        raw = await self._read(self._require_client().hget(self.existence_key(entity), slot))
+        # redis-py 命令类型为 ResponseT（Awaitable|值 联合，同步模式兼容）；
+        # 异步 client 恒返回 Awaitable，cast 仅为收窄静态类型
+        raw = await self._read(
+            cast(
+                "Awaitable[str | None]",
+                self._require_client().hget(self.existence_key(entity), slot),
+            )
+        )
         if raw is None:
             return None
         return parse_summary(str(raw))
@@ -154,9 +161,12 @@ class RedisExistenceCache:
         return bool(await self._read(self._require_client().exists(self.existence_key(entity))))
 
     async def get_entity_fields(self, entity: str) -> dict[str, dict[str, object]] | None:
-        raw: dict[str, str] = await self._read(
-            self._require_client().hgetall(self.existence_key(entity))
-        )  # type: ignore[assignment]
+        raw = await self._read(
+            cast(
+                "Awaitable[dict[str, str]]",
+                self._require_client().hgetall(self.existence_key(entity)),
+            )
+        )
         if not raw:
             return None  # Redis 中不存在只有 0 个 field 的 HASH，空 dict 即 key 不存在
         return {

streamgate-0.3.0/.import_linter_cache/a1215eee6d5c7b9f11de3767d6965c4a9c703b3e.data.json

@@ -1 +0,0 @@
-{"imports_by_module":{"streamgate":[["streamgate.obs.metrics",false,42,"from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink"],["streamgate.consumer.classifier",false,26,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["streamgate.consumer.runner",false,36,"from streamgate.consumer.runner import ConsumerWorker"],["streamgate.protocols",false,43,"from streamgate.protocols import ("],["streamgate.ingest.gateway",false,39,"from streamgate.ingest.gateway import IngestGateway"],["streamgate.config",false,25,"from streamgate.config import BackpressureConfig, ConsumerConfig, KafkaConfig"],["streamgate.ingest.admission.in_memory",false,37,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.ingest.producer",false,40,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.consumer.dlq",false,27,"from streamgate.consumer.dlq import ("],["streamgate.resilience.backpressure",false,66,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.ingest.admission.no_admission",false,38,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.resilience.health",false,67,"from streamgate.resilience.health import ("],["streamgate.consumer.loop",false,35,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.transport.kafka",false,75,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.specs",false,73,"from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT"],["streamgate.transport.codec",false,74,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.obs.logging",false,41,"from streamgate.obs.logging import configure_logger, logger"]],"streamgate.config":[["pydantic",false,8,"from pydantic import BaseModel, field_validator"]],"streamgate.consumer":[],"streamgate.consumer.classifier":[["streamgate.protocols",false,20,"from streamgate.protocols import ErrorKind"],["asyncio",false,16,"import asyncio"],["aiokafka",false,18,"from aiokafka.errors import KafkaError"]],"streamgate.consumer.dlq":[["asyncio",false,15,"import asyncio"],["json",false,16,"import json"],["datetime",false,18,"from datetime import datetime, timezone"],["aiokafka",false,20,"from aiokafka import AIOKafkaProducer"],["streamgate.config",false,22,"from streamgate.config import KafkaConfig"],["streamgate.obs.logging",false,23,"from streamgate.obs.logging import logger"],["streamgate.protocols",false,24,"from streamgate.protocols import ErrorKind, JsonObject, RecordWriter"],["dataclasses",false,17,"from dataclasses import dataclass, field"]],"streamgate.consumer.loop":[["asyncio",false,5,"import asyncio"],["collections",false,7,"from collections.abc import Callable"],["streamgate.transport.kafka",false,37,"from streamgate.transport.kafka import KafkaConsumerService, KafkaRecord"],["streamgate.protocols",false,26,"from streamgate.protocols import ("],["aiokafka",false,11,"from aiokafka.structs import OffsetAndMetadata, TopicPartition"],["streamgate.obs.logging",false,24,"from streamgate.obs.logging import logger"],["streamgate.consumer.classifier",false,14,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig"],["streamgate.obs.metrics",false,25,"from streamgate.obs.metrics import DEFAULT_METRICS, MetricsSink"],["streamgate.specs",false,36,"from streamgate.specs import ConsumeSpec"],["streamgate.consumer.dlq",false,15,"from streamgate.consumer.dlq import ("],["aiokafka",false,10,"from aiokafka.errors import CommitFailedError"],["time",false,6,"import time"]],"streamgate.consumer.runner":[["streamgate.resilience.health",false,16,"from streamgate.resilience.health import ("],["signal",false,9,"import signal"],["streamgate.protocols",false,15,"from streamgate.protocols import ErrorClassifier, MessageCodec"],["asyncio",false,8,"import asyncio"],["streamgate.consumer.loop",false,13,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.specs",false,20,"from streamgate.specs import ConsumeSpec"],["streamgate.obs.logging",false,14,"from streamgate.obs.logging import logger"],["streamgate.transport.kafka",false,22,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.config",false,11,"from streamgate.config import ConsumerConfig, KafkaConfig"],["streamgate.consumer.dlq",false,12,"from streamgate.consumer.dlq import DlqProducer"],["streamgate.transport.codec",false,21,"from streamgate.transport.codec import JsonEnvelopeCodec"]],"streamgate.contrib":[],"streamgate.contrib._deps":[["typing",false,3,"from typing import NoReturn"]],"streamgate.contrib.http_probe":[["streamgate.contrib._deps",false,9,"from streamgate.contrib._deps import require_extra_import"],["streamgate.contrib.http_probe.probe_signal",false,12,"from streamgate.contrib.http_probe.probe_signal import ("]],"streamgate.contrib.http_probe.probe_signal":[["streamgate.protocols",false,25,"from streamgate.protocols import BackpressureSnapshot, ProbeResult"],["httpx",false,22,"import httpx"],["streamgate",false,24,"from streamgate import BackpressureConfig, logger"],["time",false,20,"import time"],["asyncio",false,19,"import asyncio"]],"streamgate.contrib.mssql_sink":[["streamgate.contrib.sql_sink",false,10,"from streamgate.contrib.sql_sink import ("]],"streamgate.contrib.redis_admission":[["streamgate.contrib.redis_admission.admission",false,11,"from streamgate.contrib.redis_admission.admission import ("],["streamgate.contrib._deps",false,8,"from streamgate.contrib._deps import require_extra_import"],["streamgate.contrib.redis_admission.config",false,20,"from streamgate.contrib.redis_admission.config import RedisConfig"],["streamgate.contrib.redis_admission.existence",false,21,"from streamgate.contrib.redis_admission.existence import ("]],"streamgate.contrib.redis_admission.admission":[["streamgate.contrib.redis_admission.config",false,19,"from streamgate.contrib.redis_admission.config import RedisConfig"],["dataclasses",false,12,"from dataclasses import dataclass"],["enum",false,13,"from enum import Enum"],["redis",false,16,"from redis.exceptions import RedisError"],["streamgate",false,18,"from streamgate import logger"],["collections",false,11,"from collections.abc import Callable"],["asyncio",false,10,"import asyncio"],["streamgate.protocols",false,21,"from streamgate.protocols import ("],["typing",false,14,"from typing import Generic"],["streamgate.contrib.redis_admission.existence",false,20,"from streamgate.contrib.redis_admission.existence import RedisExistenceCache"]],"streamgate.contrib.redis_admission.config":[["pydantic",false,6,"from pydantic import BaseModel"]],"streamgate.contrib.redis_admission.existence":[["redis",false,14,"import redis.asyncio as aioredis"],["typing",false,12,"from typing import TypeVar"],["streamgate",false,16,"from streamgate import logger"],["json",false,10,"import json"],["streamgate.contrib.redis_admission.config",false,17,"from streamgate.contrib.redis_admission.config import RedisConfig"],["collections",false,11,"from collections.abc import Awaitable"],["asyncio",false,9,"import asyncio"]],"streamgate.contrib.sql_sink":[["streamgate.contrib.sql_sink.engines",false,15,"from streamgate.contrib.sql_sink.engines import ("],["streamgate.contrib.sql_sink.classifier",false,13,"from streamgate.contrib.sql_sink.classifier import SQLAlchemyErrorClassifier"],["streamgate.contrib.sql_sink.config",false,14,"from streamgate.contrib.sql_sink.config import DbConfig"],["streamgate.contrib.sql_sink.upsert",false,19,"from streamgate.contrib.sql_sink.upsert import ("],["streamgate.contrib.sql_sink.backfill",false,12,"from streamgate.contrib.sql_sink.backfill import SqlBackfill"],["streamgate.contrib._deps",false,9,"from streamgate.contrib._deps import require_extra_import"]],"streamgate.contrib.sql_sink._dialects":[],"streamgate.contrib.sql_sink._dialects.mssql":[["sqlmodel",false,19,"from sqlmodel import SQLModel"],["sqlalchemy",false,18,"from sqlalchemy.dialects import mssql"],["collections",false,14,"from collections.abc import Iterator"],["streamgate.protocols",false,21,"from streamgate.protocols import JsonObject"],["sqlalchemy",false,16,"from sqlalchemy import String, TextClause, text"],["sqlalchemy",false,17,"from sqlalchemy import inspect as sa_inspect"]],"streamgate.contrib.sql_sink._dialects.sqlite":[["sqlalchemy",false,4,"from sqlalchemy.dialects.sqlite import insert as sqlite_insert"],["sqlmodel",false,6,"from sqlmodel import SQLModel"],["sqlalchemy",false,5,"from sqlalchemy.ext.asyncio import AsyncSession"],["sqlalchemy",false,3,"from sqlalchemy import text"],["streamgate.protocols",false,8,"from streamgate.protocols import JsonObject"]],"streamgate.contrib.sql_sink.backfill":[["streamgate",false,13,"from streamgate import logger"],["sqlalchemy",false,11,"from sqlalchemy.ext.asyncio import AsyncEngine"],["datetime",false,8,"from datetime import datetime"],["sqlalchemy",false,10,"from sqlalchemy import text"],["asyncio",false,7,"import asyncio"],["streamgate.protocols",false,14,"from streamgate.protocols import JsonObject"]],"streamgate.contrib.sql_sink.classifier":[["sqlalchemy",false,23,"from sqlalchemy.exc import ("],["sqlalchemy",false,28,"from sqlalchemy.exc import ("],["streamgate",false,32,"from streamgate import ErrorKind"],["collections",false,20,"from collections.abc import Iterator"],["asyncio",false,18,"import asyncio"],["re",false,19,"import re"],["aiokafka",false,22,"from aiokafka.errors import KafkaError"]],"streamgate.contrib.sql_sink.config":[["pydantic",false,7,"from pydantic import BaseModel"]],"streamgate.contrib.sql_sink.engines":[["sqlalchemy",false,3,"from sqlalchemy import event"],["sqlalchemy",false,4,"from sqlalchemy.ext.asyncio import ("],["streamgate.contrib.sql_sink.config",false,12,"from streamgate.contrib.sql_sink.config import DbConfig"],["streamgate",false,11,"from streamgate import logger"]],"streamgate.contrib.sql_sink.upsert":[["streamgate.contrib.sql_sink._dialects.mssql",false,19,"from streamgate.contrib.sql_sink._dialects.mssql import mssql_cast_types"],["streamgate.protocols",false,21,"from streamgate.protocols import JsonObject, WriteResult"],["collections",false,9,"from collections.abc import Callable"],["asyncio",false,8,"import asyncio"],["streamgate.contrib.sql_sink.config",false,20,"from streamgate.contrib.sql_sink.config import DbConfig"],["sqlmodel",false,14,"from sqlmodel import SQLModel"],["sqlalchemy",false,11,"from sqlalchemy import inspect as sa_inspect"],["streamgate.contrib.sql_sink._dialects.mssql",false,17,"from streamgate.contrib.sql_sink._dialects import mssql as mssql_dialect"],["sqlalchemy",false,13,"from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker"],["streamgate",false,16,"from streamgate import logger"],["sqlalchemy",false,12,"from sqlalchemy import text"],["streamgate.contrib.sql_sink._dialects.sqlite",false,18,"from streamgate.contrib.sql_sink._dialects import sqlite as sqlite_dialect"]],"streamgate.contrib.sqlite_sink":[["streamgate.contrib.sql_sink",false,9,"from streamgate.contrib.sql_sink import ("]],"streamgate.ingest":[],"streamgate.ingest.admission":[],"streamgate.ingest.admission.in_memory":[["collections",false,12,"from collections.abc import Callable"],["typing",false,13,"from typing import Generic"],["streamgate.protocols",false,15,"from streamgate.protocols import Decision, JsonObject, RecordT"]],"streamgate.ingest.admission.no_admission":[["typing",false,3,"from typing import Generic"],["streamgate.protocols",false,5,"from streamgate.protocols import Decision, RecordT"]],"streamgate.ingest.gateway":[["streamgate.protocols",false,16,"from streamgate.protocols import ("],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"],["typing",false,9,"from typing import Generic"],["streamgate.config",false,11,"from streamgate.config import BackpressureConfig, KafkaConfig"],["streamgate.ingest.producer",false,14,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.resilience.health",false,26,"from streamgate.resilience.health import ("],["streamgate.transport.codec",false,31,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.ingest.admission.in_memory",false,12,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.specs",false,30,"from streamgate.specs import IngestBinding, IngestRecordT"],["streamgate.ingest.admission.no_admission",false,13,"from streamgate.ingest.admission.no_admission import NoAdmission"],["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.resilience.backpressure",false,25,"from streamgate.resilience.backpressure import ManualBackpressureSignal"]],"streamgate.ingest.producer":[["json",false,9,"import json"],["streamgate.obs.logging",false,17,"from streamgate.obs.logging import logger"],["aiokafka",false,13,"from aiokafka import AIOKafkaProducer"],["aiokafka",false,14,"from aiokafka.errors import KafkaConnectionError, KafkaError"],["asyncio",false,8,"import asyncio"],["time",false,10,"import time"],["datetime",false,11,"from datetime import datetime, timezone"],["streamgate.protocols",false,18,"from streamgate.protocols import JsonObject"],["streamgate.config",false,16,"from streamgate.config import KafkaConfig"]],"streamgate.obs":[],"streamgate.obs.logging":[["loguru",false,6,"from loguru._handler import Message"],["datetime",false,3,"from datetime import timezone"],["loguru",false,5,"from loguru import logger as _logger"],["sys",false,2,"import sys"],["json",false,1,"import json"]],"streamgate.obs.metrics":[["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"],["typing",false,8,"from typing import Protocol"]],"streamgate.protocols":[["typing",false,11,"from typing import Protocol, TypeVar, runtime_checkable"],["collections",false,7,"from collections.abc import Awaitable, Callable"],["datetime",false,9,"from datetime import datetime"],["enum",false,10,"from enum import Enum"],["dataclasses",false,8,"from dataclasses import dataclass, field"]],"streamgate.resilience":[],"streamgate.resilience.backpressure":[["streamgate.protocols",false,10,"from streamgate.protocols import AllowAllSignal, BackpressureSnapshot"]],"streamgate.resilience.health":[["datetime",false,10,"from datetime import datetime, timezone"],["asyncio",false,7,"import asyncio"],["typing",false,11,"from typing import Protocol"],["pydantic",false,13,"from pydantic import BaseModel, Field"],["collections",false,9,"from collections.abc import Awaitable"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"],["time",false,8,"import time"]],"streamgate.specs":[["collections",false,10,"from collections.abc import Callable, Hashable"],["streamgate.protocols",false,16,"from streamgate.protocols import ("],["pydantic",false,14,"from pydantic import BaseModel"],["typing",false,12,"from typing import Generic, Literal, TypeVar"],["dataclasses",false,11,"from dataclasses import dataclass, field"]],"streamgate.transport":[],"streamgate.transport.codec":[["streamgate.protocols",false,11,"from streamgate.protocols import Envelope, JsonObject"],["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"],["json",false,8,"import json"]],"streamgate.transport.kafka":[["asyncio",false,3,"import asyncio"],["aiokafka",false,7,"from aiokafka.errors import CommitFailedError, KafkaConnectionError, KafkaError"],["aiokafka",false,5,"from aiokafka import AIOKafkaConsumer"],["aiokafka",false,8,"from aiokafka.structs import ConsumerRecord, OffsetAndMetadata, TopicPartition"],["streamgate.config",false,10,"from streamgate.config import ConsumerConfig, KafkaConfig"],["aiokafka",false,6,"from aiokafka.consumer.group_coordinator import GroupCoordinator"],["streamgate.obs.logging",false,11,"from streamgate.obs.logging import logger"]]},"version":2}

streamgate-0.3.0/.import_linter_cache/streamgate.meta.json

@@ -1 +0,0 @@
-{"streamgate.contrib.sql_sink.backfill": 1788835942.310305, "streamgate.consumer.classifier": 1788778605.4753034, "streamgate.obs.logging": 1788749262.353655, "streamgate.obs.metrics": 1788749264.9746833, "streamgate.contrib.redis_admission.config": 1788835837.2772365, "streamgate": 1788836110.1479475, "streamgate.contrib.redis_admission": 1788835898.0475388, "streamgate.contrib._deps": 1788835802.9044309, "streamgate.consumer": 1788742270.5761602, "streamgate.contrib.sql_sink.config": 1788835924.6027458, "streamgate.ingest.admission.no_admission": 1788759910.7492158, "streamgate.contrib.sql_sink.engines": 1788835942.286183, "streamgate.contrib.http_probe": 1788836174.9308388, "streamgate.consumer.runner": 1788778819.43872, "streamgate.consumer.dlq": 1788778795.1676013, "streamgate.contrib.sql_sink._dialects.sqlite": 1788835977.7374504, "streamgate.config": 1788778673.809221, "streamgate.contrib.sqlite_sink": 1788836174.9292364, "streamgate.contrib": 1788835802.905441, "streamgate.resilience.health": 1788767769.120218, "streamgate.transport": 1788742270.5761602, "streamgate.contrib.sql_sink._dialects": 1788835977.7374504, "streamgate.ingest": 1788742270.5761602, "streamgate.contrib.http_probe.probe_signal": 1788836090.470792, "streamgate.contrib.redis_admission.admission": 1788835898.0445402, "streamgate.contrib.redis_admission.existence": 1788835837.2841742, "streamgate.protocols": 1788780004.1189399, "streamgate.resilience": 1788742270.5761602, "streamgate.transport.kafka": 1788776206.140225, "streamgate.specs": 1788778652.1194282, "streamgate.ingest.admission.in_memory": 1788778617.9172873, "streamgate.ingest.producer": 1788776206.139223, "streamgate.consumer.loop": 1788778765.328234, "streamgate.ingest.gateway": 1788778851.3711455, "streamgate.contrib.mssql_sink": 1788836174.9297483, "streamgate.contrib.sql_sink": 1788836043.336221, "streamgate.contrib.sql_sink._dialects.mssql": 1788836025.5239024, "streamgate.obs": 1788742270.5761602, "streamgate.contrib.sql_sink.classifier": 1788835977.7154298, "streamgate.ingest.admission": 1788742270.5761602, "streamgate.contrib.sql_sink.upsert": 1788836025.5259016, "streamgate.resilience.backpressure": 1788778626.931601, "streamgate.transport.codec": 1788749619.0487723}