streamgate 0.4.0__tar.gz → 0.5.0__tar.gz

{streamgate-0.4.0 → streamgate-0.5.0}/CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.0] - 2026-09-10
+
+### Added
+
+- **`AdmissionPolicy` gains `on_send_success` / `on_send_failed` hooks.**
+  `on_send_success(record)` fires after every broker-acked Kafka send
+  (overwrite path included); `on_send_failed(record)` fires when a send
+  fails after the admission reservation was placed. The framework default
+  (no-op) preserves current behavior — the placeholder stays reserved and
+  self-heals via TTL/backfill — but implementations can now release the
+  reservation to make the key immediately re-submittable, accepting the
+  duplicate risk of ambiguous (timeout) failures.
+
+### Changed
+
+- **`AdmissionPolicy.on_accepted` renamed to `on_overwrite_accepted`.**
+  The old hook only fired on the overwrite path (not on every accepted
+  send, despite the name); the new name states that. Custom policies that
+  implement `on_accepted` keep working: the gateway falls back to it when
+  `on_overwrite_accepted` is absent. Built-in policies provide both
+  (the old name delegates to the new one).
+
 ## [0.4.0] - 2026-09-08
 
 ### Added
@@ -214,5 +236,7 @@ First public release. Data pipeline framework: conditional admission → reliabl
   - Mechanisms as core dependencies; all policy carriers (DB drivers, redis, httpx) as optional extras with lazy loading and install-guidance errors.
   - Typed configuration objects mapping 1:1 to `KAFKA__*` / `CONSUMER__*` / `DB__*` / `REDIS__*` / `BACKPRESSURE__*` environment variables; required settings fail fast at startup.
 
+[0.5.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.5.0
+[0.4.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.4.0
 [0.2.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.2.0
 [0.1.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.1.0

{streamgate-0.4.0 → streamgate-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: streamgate
-Version: 0.4.0
+Version: 0.5.0
 Summary: A pure-core Kafka data pipeline framework: conditional admission, reliable delivery, pluggable sinks via typed hooks.
 Project-URL: Homepage, https://github.com/pwg-code/streamgate
 Project-URL: Repository, https://github.com/pwg-code/streamgate

{streamgate-0.4.0 → streamgate-0.5.0}/README.zh-CN.md

@@ -173,7 +173,11 @@ worker = ConsumerWorker(spec, ..., error_classifier=MyClassifier())
 ```python
 class MyAdmission:
     async def admit(self, record, *, overwrite=False): ...  # 判定：允许/冲突/拒绝
-    async def on_accepted(self, record): ...                # Kafka 发送成功后
+    async def on_send_success(self, record): ...            # 每次 Kafka 发送成功后（通知型）
+    async def on_send_failed(self, record): ...             # 发送失败后；默认保留占位自愈，
+                                                            # 也可在此释放占位换"立即可重发"
+    async def on_overwrite_accepted(self, record): ...      # 仅 overwrite：摘要写，返回 False
+                                                            # → cache_updated=false
     async def on_persisted(self, record): ...               # 消费落库成功后刷新
     # 另有 start / close / 健康探测方法，见 protocols.py
 

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/__init__.py

@@ -79,7 +79,7 @@ from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT
 from streamgate.transport.codec import JsonEnvelopeCodec
 from streamgate.transport.kafka import KafkaConsumerService
 
-__version__ = "0.4.0"
+__version__ = "0.5.0"
 
 __all__ = [
     "AdmissionPolicy",

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/contrib/redis_admission/admission.py

@@ -3,8 +3,9 @@
 协议适配层：entity+slot 原子占位（Lua）、idle-GC TTL、
 空实体哨兵、fail-closed、overwrite 预检；TTL/前缀/降级开关/错误码全参数化。
 
-框架保证调用时序：admit → (Kafka 发送) → on_accepted；消费侧
-write 成功 → on_persisted（准入与消费侧的唯一耦合点）。
+框架保证调用时序：admit → (Kafka 发送) → on_send_success（每次成功，通知型）/
+on_send_failed（每次失败，默认保留占位 TTL 自愈）→ on_overwrite_accepted（仅
+overwrite 路径）；消费侧 write 成功 → on_persisted（准入与消费侧的唯一耦合点）。
 """
 
 import asyncio
@@ -197,7 +198,16 @@ class RedisExistenceAdmission(Generic[RecordT]):
             return Decision.allow()
         return result
 
-    async def on_accepted(self, record: RecordT) -> bool:
+    async def on_send_success(self, record: RecordT) -> None:
+        """通知型钩子：占位已在 admit 原子写入，无需动作。"""
+        return None
+
+    async def on_send_failed(self, record: RecordT) -> None:
+        """默认保留占位（TTL 过后冷路径回源自愈）；需"失败立即可重发"可在此
+        删除 existence key 的 field（自担模糊失败重复风险）。"""
+        return None
+
+    async def on_overwrite_accepted(self, record: RecordT) -> bool:
         """overwrite 摘要写：失败重试 1 次；仍失败 ERROR（供告警删 key）并返回 False。"""
         entity = str(self._entity_key(record))
         slot = str(self._slot_key(record))
@@ -215,6 +225,10 @@ class RedisExistenceAdmission(Generic[RecordT]):
                     )
         return False
 
+    async def on_accepted(self, record: RecordT) -> bool:
+        """向后兼容别名：等价 on_overwrite_accepted。"""
+        return await self.on_overwrite_accepted(record)
+
     async def on_persisted(self, record: RecordT) -> None:
         """落库成功 → 提交 offset 前的权威缓存刷新（失败 WARN 不影响提交）。"""
         entity = str(self._entity_key(record))

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/ingest/admission/in_memory.py

@@ -18,8 +18,9 @@ from streamgate.protocols import Decision, JsonObject, RecordT
 class InMemoryAdmission(Generic[RecordT]):
     """唯一性契约的进程内实现：检查 + 原子占位 + 摘要刷新。
 
-    框架保证调用时序：admit → (Kafka 发送) → on_accepted（仅 overwrite 路径）；
-    消费侧落库成功 → on_persisted（权威刷新）。
+    框架保证调用时序：admit → (Kafka 发送) → on_send_success（每次成功，通知型）/
+    on_send_failed（每次失败，默认保留占位自愈）→ on_overwrite_accepted（仅
+    overwrite 路径）；消费侧落库成功 → on_persisted（权威刷新）。
     """
 
     def __init__(
@@ -37,7 +38,7 @@ class InMemoryAdmission(Generic[RecordT]):
 
     async def admit(self, record: RecordT, *, overwrite: bool = False) -> Decision:
         if overwrite:
-            # 409 确认后的完整重发：跳过唯一性判定（摘要由 on_accepted 刷新）
+            # 409 确认后的完整重发：跳过唯一性判定（摘要由 on_overwrite_accepted 刷新）
             return Decision.allow()
         key = (str(self._entity_key(record)), str(self._slot_key(record)))
         existing = self._store.get(key)
@@ -46,13 +47,25 @@ class InMemoryAdmission(Generic[RecordT]):
         self._store[key] = self._summary(record)
         return Decision.allow()
 
-    async def on_accepted(self, record: RecordT) -> bool:
+    async def on_send_success(self, record: RecordT) -> None:
+        """通知型钩子：占位已在 admit 写入，无需动作。"""
+        return None
+
+    async def on_send_failed(self, record: RecordT) -> None:
+        """默认保留占位（自愈）；需"失败立即可重发"可在此删除 _store 键。"""
+        return None
+
+    async def on_overwrite_accepted(self, record: RecordT) -> bool:
         """overwrite 路径摘要写（内存操作恒成功）。"""
         self._store[(str(self._entity_key(record)), str(self._slot_key(record)))] = (
             self._summary(record)
         )
         return True
 
+    async def on_accepted(self, record: RecordT) -> bool:
+        """向后兼容别名：等价 on_overwrite_accepted。"""
+        return await self.on_overwrite_accepted(record)
+
     async def on_persisted(self, record: RecordT) -> None:
         """落库成功后的权威摘要刷新（幂等）。"""
         self._store[(str(self._entity_key(record)), str(self._slot_key(record)))] = (

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/ingest/admission/no_admission.py

@@ -11,9 +11,19 @@ class NoAdmission(Generic[RecordT]):
     async def admit(self, record: RecordT, *, overwrite: bool = False) -> Decision:
         return Decision.allow()
 
-    async def on_accepted(self, record: RecordT) -> bool:
+    async def on_send_success(self, record: RecordT) -> None:
+        return None
+
+    async def on_send_failed(self, record: RecordT) -> None:
+        return None
+
+    async def on_overwrite_accepted(self, record: RecordT) -> bool:
         return True
 
+    async def on_accepted(self, record: RecordT) -> bool:
+        """向后兼容别名：等价 on_overwrite_accepted。"""
+        return await self.on_overwrite_accepted(record)
+
     async def on_persisted(self, record: RecordT) -> None:
         return None
 

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/ingest/gateway.py

@@ -1,4 +1,4 @@
-"""IngestGateway：传输无关的接收编排内核（背压 → 准入 → 发送 → on_accepted）。
+"""IngestGateway：传输无关的接收编排内核（背压 → 准入 → 发送 → 发送结果钩子）。
 
 HTTP 鉴权/路由/OpenAPI 等呈现职责归使用方适配层：process() 返回传输无关的
 IngestOutcome，不抛 HTTP 异常、不构造 Response。日志事件名与字段为稳定观测
@@ -6,8 +6,9 @@ IngestOutcome，不抛 HTTP 异常、不构造 Response。日志事件名与字
 """
 
 import time
+from collections.abc import Awaitable, Callable
 from datetime import datetime, timezone
-from typing import Generic
+from typing import Generic, cast
 
 from streamgate.config import BackpressureConfig, KafkaConfig, MetricsConfig
 from streamgate.ingest.admission.in_memory import InMemoryAdmission
@@ -37,6 +38,16 @@ def _iso_z(dt: datetime) -> str:
     return dt.isoformat().replace("+00:00", "Z")
 
 
+def _optional_hook(
+    admission: AdmissionPolicy[IngestRecordT], name: str
+) -> Callable[[IngestRecordT], Awaitable[object]] | None:
+    """按名解析可选钩子：旧版策略缺新钩子时返回 None（向后兼容）。"""
+    candidate: object | None = getattr(admission, name, None)
+    if not callable(candidate):
+        return None
+    return cast("Callable[[IngestRecordT], Awaitable[object]]", candidate)
+
+
 class IngestGateway(Generic[IngestRecordT]):
     """接收机制内核：start/close 管资源生命周期，process 跑单条接收链路。"""
 
@@ -141,7 +152,7 @@ class IngestGateway(Generic[IngestRecordT]):
         overwrite: bool | None = None,
         source: str = "unknown",
     ) -> IngestOutcome:
-        """接收单条记录：背压 → 准入 → Kafka 发送 → on_accepted。
+        """接收单条记录：背压 → 准入 → Kafka 发送 → 发送结果钩子。
 
         overwrite=None 时按 binding.is_overwrite(record) 解析。
         未 start() 直接抛 RuntimeError（装配错误，非运行时降级）。
@@ -247,9 +258,9 @@ class IngestGateway(Generic[IngestRecordT]):
         source: str,
         log_ctx: JsonObject,
     ) -> IngestOutcome:
-        """Kafka 发送（失败 502；占位保留自愈）→ overwrite 摘要写 → 成功日志。"""
+        """Kafka 发送 → 失败 502（on_send_failed，默认占位保留自愈）→ 成功钩子
+        → overwrite 摘要写 → 成功日志。"""
         binding = self.binding
-        producer = self._require_producer()
         entity = str(binding.entity_key(record))
         slot = str(binding.slot_key(record))
         key = (
@@ -263,30 +274,17 @@ class IngestGateway(Generic[IngestRecordT]):
             _iso_z(received_at),
             source,
         )
-        t0 = time.monotonic()
-        try:
-            await producer.send(key=key, message=message)
-        except Exception as e:
-            self._metrics.produce_failure.record()
-            logger.error(
-                "kafka_send_failed",
-                **log_ctx,
-                error=str(e),
-                source=source,
-                error_code=binding.kafka_unavailable_code,
-            )
+        if not await self._produce(
+            record, key=key, message=message, log_ctx=log_ctx, source=source
+        ):
             return IngestOutcome.kafka_unavailable(
                 error_code=binding.kafka_unavailable_code,
                 detail=binding.kafka_unavailable_detail,
             )
-        self._metrics.produce_success.record()
-        self._metrics.produce_latency.record_latency(
-            (time.monotonic() - t0) * 1000.0  # send 调用到 broker 确认耗时（毫秒）
+        cache_updated = (
+            await self._overwrite_summary_write(record) if overwrite else True
         )
 
-        # ---- overwrite 路径：Kafka 成功后幂等摘要写（实现内自定重试）----
-        cache_updated = await self._admission.on_accepted(record) if overwrite else True
-
         logger.info(
             "ingest_request",
             **(
@@ -300,6 +298,68 @@ class IngestGateway(Generic[IngestRecordT]):
         )
         return IngestOutcome.accepted(received_at, cache_updated=cache_updated)
 
+    async def _produce(
+        self,
+        record: IngestRecordT,
+        *,
+        key: str,
+        message: JsonObject,
+        log_ctx: JsonObject,
+        source: str,
+    ) -> bool:
+        """Kafka 发送 + 指标 + 发送结果钩子。False = 失败（已记录日志与钩子）。"""
+        producer = self._require_producer()
+        t0 = time.monotonic()
+        try:
+            await producer.send(key=key, message=message)
+        except Exception as e:
+            self._metrics.produce_failure.record()
+            logger.error(
+                "kafka_send_failed",
+                **log_ctx,
+                error=str(e),
+                source=source,
+                error_code=self.binding.kafka_unavailable_code,
+            )
+            await self._notify_send_failed(record)
+            return False
+        self._metrics.produce_success.record()
+        self._metrics.produce_latency.record_latency(
+            (time.monotonic() - t0) * 1000.0  # send 调用到 broker 确认耗时（毫秒）
+        )
+        await self._notify_send_success(record)
+        return True
+
+    async def _notify_send_success(self, record: IngestRecordT) -> None:
+        """每次发送成功的通知钩子（best-effort：钩子异常不影响已成功的请求）。"""
+        hook = _optional_hook(self._admission, "on_send_success")
+        if hook is None:
+            return
+        try:
+            await hook(record)
+        except Exception as e:
+            logger.warning("admission_send_success_hook_failed", error=str(e))
+
+    async def _notify_send_failed(self, record: IngestRecordT) -> None:
+        """发送失败钩子（best-effort：默认保留占位，实现可释放换取立即重发）。"""
+        hook = _optional_hook(self._admission, "on_send_failed")
+        if hook is None:
+            return
+        try:
+            await hook(record)
+        except Exception as e:
+            logger.warning("admission_send_failed_hook_failed", error=str(e))
+
+    async def _overwrite_summary_write(self, record: IngestRecordT) -> bool:
+        """overwrite 路径摘要写；旧版策略回退 on_accepted（向后兼容）。"""
+        hook = _optional_hook(self._admission, "on_overwrite_accepted")
+        if hook is None:
+            legacy = _optional_hook(self._admission, "on_accepted")
+            if legacy is None:
+                return True
+            return bool(await legacy(record))
+        return bool(await hook(record))
+
     def _require_producer(self) -> KafkaProducerService:
         """start() 完成前不可达；与 KafkaProducerService 未启动语义一致。"""
         if self._producer is None:

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/protocols.py

@@ -97,9 +97,21 @@ class AdmissionPolicy(Protocol[RecordT]):
         overwrite=True 时策略应跳过唯一性判定（409 确认后的完整重发）。"""
         ...
 
-    async def on_accepted(self, record: RecordT) -> bool:
-        """Kafka 发送成功后：占位/摘要写（失败语义由实现自定）。
-        返回 False 表示"缓存未反映本次记录"（映射到 cache_updated=false）。"""
+    async def on_send_success(self, record: RecordT) -> None:
+        """每次 Kafka 发送成功（broker ack）后调用（含 overwrite 路径；通知型）。
+        非 overwrite 路径占位已在 admit 写入，默认无需动作；实现可做缓存续期等。"""
+        ...
+
+    async def on_send_failed(self, record: RecordT) -> None:
+        """Kafka 发送失败后调用（此时占位可能已写入）。
+        框架默认语义是保留占位（防模糊失败重复，靠 TTL/回源自愈）；
+        实现可在此释放占位换取"立即可重发"，自担重复风险。"""
+        ...
+
+    async def on_overwrite_accepted(self, record: RecordT) -> bool:
+        """仅 overwrite 路径、Kafka 成功后：幂等摘要写（失败语义由实现自定）。
+        返回 False 表示"缓存未反映本次记录"（映射到 cache_updated=false）。
+        兼容：旧版策略只实现 on_accepted 时，框架按本钩子语义回退调用。"""
         ...
 
     async def on_persisted(self, record: RecordT) -> None:

{streamgate-0.4.0 → streamgate-0.5.0}/src/streamgate/specs.py

@@ -27,7 +27,8 @@ IngestRecordT = TypeVar("IngestRecordT", bound=BaseModel)
 @dataclass
 class IngestBinding(Generic[IngestRecordT]):
     """接收侧机制绑定：使用方完成 Schema 校验后，网关执行
-    背压 → 准入 → Kafka 发送 → on_accepted 编排。
+    背压 → 准入 → Kafka 发送 → 发送结果钩子（on_send_success/on_send_failed/
+    on_overwrite_accepted）编排。
 
     路由/URL/鉴权/OpenAPI/响应模型等 HTTP 呈现职责归使用方适配层，
     本声明只承载机制所需的策略钩子与错误码契约。