streamgate 0.3.1__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {streamgate-0.3.1 → streamgate-0.5.0}/AGENTS.md +7 -0
  2. {streamgate-0.3.1 → streamgate-0.5.0}/CHANGELOG.md +54 -0
  3. {streamgate-0.3.1 → streamgate-0.5.0}/CONFIGURATION.md +14 -0
  4. {streamgate-0.3.1 → streamgate-0.5.0}/PKG-INFO +3 -2
  5. {streamgate-0.3.1 → streamgate-0.5.0}/README.md +2 -1
  6. {streamgate-0.3.1 → streamgate-0.5.0}/README.zh-CN.md +7 -2
  7. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/__init__.py +8 -2
  8. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/config.py +103 -87
  9. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/consumer/dlq.py +374 -374
  10. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/consumer/loop.py +679 -656
  11. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/consumer/runner.py +15 -1
  12. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/redis_admission/admission.py +17 -3
  13. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/admission/in_memory.py +17 -4
  14. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/admission/no_admission.py +47 -37
  15. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/gateway.py +97 -20
  16. streamgate-0.5.0/src/streamgate/obs/metrics.py +148 -0
  17. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/protocols.py +401 -389
  18. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/resilience/health.py +103 -3
  19. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/specs.py +2 -1
  20. streamgate-0.3.1/.import_linter_cache/.gitignore +0 -2
  21. streamgate-0.3.1/.import_linter_cache/5780e4798d429798e55d962d9844c38a9b5e0494.data.json +0 -1
  22. streamgate-0.3.1/.import_linter_cache/CACHEDIR.TAG +0 -3
  23. streamgate-0.3.1/.import_linter_cache/a1215eee6d5c7b9f11de3767d6965c4a9c703b3e.data.json +0 -1
  24. streamgate-0.3.1/.import_linter_cache/streamgate.meta.json +0 -1
  25. streamgate-0.3.1/src/streamgate/obs/metrics.py +0 -30
  26. {streamgate-0.3.1 → streamgate-0.5.0}/.github/workflows/ci.yml +0 -0
  27. {streamgate-0.3.1 → streamgate-0.5.0}/.github/workflows/release.yml +0 -0
  28. {streamgate-0.3.1 → streamgate-0.5.0}/.gitignore +0 -0
  29. {streamgate-0.3.1 → streamgate-0.5.0}/CONTRIBUTING.md +0 -0
  30. {streamgate-0.3.1 → streamgate-0.5.0}/LICENSE +0 -0
  31. {streamgate-0.3.1 → streamgate-0.5.0}/examples/docker-compose.yml +0 -0
  32. {streamgate-0.3.1 → streamgate-0.5.0}/examples/http_probe/README.md +0 -0
  33. {streamgate-0.3.1 → streamgate-0.5.0}/examples/http_probe/consume.py +0 -0
  34. {streamgate-0.3.1 → streamgate-0.5.0}/examples/http_probe/health_server.py +0 -0
  35. {streamgate-0.3.1 → streamgate-0.5.0}/examples/http_probe/models.py +0 -0
  36. {streamgate-0.3.1 → streamgate-0.5.0}/examples/http_probe/produce.py +0 -0
  37. {streamgate-0.3.1 → streamgate-0.5.0}/examples/prod_pipeline/README.md +0 -0
  38. {streamgate-0.3.1 → streamgate-0.5.0}/examples/prod_pipeline/consume.py +0 -0
  39. {streamgate-0.3.1 → streamgate-0.5.0}/examples/prod_pipeline/health_server.py +0 -0
  40. {streamgate-0.3.1 → streamgate-0.5.0}/examples/prod_pipeline/models.py +0 -0
  41. {streamgate-0.3.1 → streamgate-0.5.0}/examples/prod_pipeline/produce.py +0 -0
  42. {streamgate-0.3.1 → streamgate-0.5.0}/examples/pure_pipeline/README.md +0 -0
  43. {streamgate-0.3.1 → streamgate-0.5.0}/examples/pure_pipeline/consume.py +0 -0
  44. {streamgate-0.3.1 → streamgate-0.5.0}/examples/pure_pipeline/models.py +0 -0
  45. {streamgate-0.3.1 → streamgate-0.5.0}/examples/pure_pipeline/produce.py +0 -0
  46. {streamgate-0.3.1 → streamgate-0.5.0}/examples/redis_admission/README.md +0 -0
  47. {streamgate-0.3.1 → streamgate-0.5.0}/examples/redis_admission/models.py +0 -0
  48. {streamgate-0.3.1 → streamgate-0.5.0}/examples/redis_admission/produce.py +0 -0
  49. {streamgate-0.3.1 → streamgate-0.5.0}/examples/sqlite_sink/README.md +0 -0
  50. {streamgate-0.3.1 → streamgate-0.5.0}/examples/sqlite_sink/consume.py +0 -0
  51. {streamgate-0.3.1 → streamgate-0.5.0}/examples/sqlite_sink/models.py +0 -0
  52. {streamgate-0.3.1 → streamgate-0.5.0}/examples/sqlite_sink/produce.py +0 -0
  53. {streamgate-0.3.1 → streamgate-0.5.0}/pyproject.toml +0 -0
  54. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/consumer/__init__.py +0 -0
  55. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/consumer/classifier.py +0 -0
  56. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/__init__.py +0 -0
  57. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/_deps.py +0 -0
  58. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/http_probe/__init__.py +0 -0
  59. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/http_probe/probe_signal.py +0 -0
  60. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/mssql_sink/__init__.py +0 -0
  61. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/redis_admission/__init__.py +0 -0
  62. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/redis_admission/config.py +0 -0
  63. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/redis_admission/existence.py +0 -0
  64. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/__init__.py +0 -0
  65. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/_dialects/__init__.py +0 -0
  66. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/_dialects/mssql.py +0 -0
  67. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/_dialects/sqlite.py +0 -0
  68. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/backfill.py +0 -0
  69. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/classifier.py +0 -0
  70. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/config.py +0 -0
  71. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/engines.py +0 -0
  72. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sql_sink/upsert.py +0 -0
  73. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/contrib/sqlite_sink/__init__.py +0 -0
  74. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/__init__.py +0 -0
  75. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/admission/__init__.py +0 -0
  76. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/ingest/producer.py +0 -0
  77. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/obs/__init__.py +0 -0
  78. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/obs/logging.py +0 -0
  79. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/resilience/__init__.py +0 -0
  80. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/resilience/backpressure.py +0 -0
  81. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/transport/__init__.py +0 -0
  82. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/transport/codec.py +0 -0
  83. {streamgate-0.3.1 → streamgate-0.5.0}/src/streamgate/transport/kafka.py +0 -0
  84. {streamgate-0.3.1 → streamgate-0.5.0}/uv.lock +0 -0
@@ -10,3 +10,10 @@
10
10
  - 禁止手动逐字段映射:用 `model_validate()`(ORM 对象加 `from_attributes=True`)代替 `_to_response()` 和 `Schema(**obj)`;用 `model_dump(exclude=.../exclude_unset=True)` 代替手写字典;字段定义只出现在 Schema 一处。响应体 Schema 有默认值,请求体 Schema 保持必填。
11
11
  - 新增代码必须通过 pyright 静态检查。
12
12
 
13
+ ## 发布流程(PyPI)
14
+
15
+ - PyPI Trusted Publisher 已配置(owner=pwg-code, repo=streamgate, workflow=release.yml, environment=pypi)。发布一律走 GitHub Actions 自动流程,禁止本地 `uv publish`/`twine upload`。
16
+ - 步骤:更新 `src/streamgate/__init__.py` 的 `__version__` 与 `CHANGELOG.md`(两者必须与 tag 一致,workflow 会校验)→ commit → `git tag vX.Y.Z` → `git push origin main vX.Y.Z` → tag 推送自动触发 `.github/workflows/release.yml` 完成构建与发布。
17
+ - 本地只做 `uv build` 验证,不承担发布职责。
18
+
19
+
@@ -5,6 +5,58 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.5.0] - 2026-09-10
9
+
10
+ ### Added
11
+
12
+ - **`AdmissionPolicy` gains `on_send_success` / `on_send_failed` hooks.**
13
+ `on_send_success(record)` fires after every broker-acked Kafka send
14
+ (overwrite path included); `on_send_failed(record)` fires when a send
15
+ fails after the admission reservation was placed. The framework default
16
+ (no-op) preserves current behavior — the placeholder stays reserved and
17
+ self-heals via TTL/backfill — but implementations can now release the
18
+ reservation to make the key immediately re-submittable, accepting the
19
+ duplicate risk of ambiguous (timeout) failures.
20
+
21
+ ### Changed
22
+
23
+ - **`AdmissionPolicy.on_accepted` renamed to `on_overwrite_accepted`.**
24
+ The old hook only fired on the overwrite path (not on every accepted
25
+ send, despite the name); the new name states that. Custom policies that
26
+ implement `on_accepted` keep working: the gateway falls back to it when
27
+ `on_overwrite_accepted` is absent. Built-in policies provide both
28
+ (the old name delegates to the new one).
29
+
30
+ ## [0.4.0] - 2026-09-08
31
+
32
+ ### Added
33
+
34
+ - **Health snapshots now carry ops rate/latency metrics.** Both
35
+ `IngestHealthResponse` and `ConsumerHealthResponse` gain sliding-window
36
+ fields (all defaulting to `0.0`, computed on read over a 1-second bucket
37
+ sliding window — no background tasks, no extra endpoint):
38
+ - ingest: `receive_rate`, `admission_conflict_rate`,
39
+ `backpressure_reject_rate`, `produce_success_rate`,
40
+ `produce_failure_rate`, `produce_latency_ms_avg`,
41
+ `produce_latency_ms_max`;
42
+ - consume: `consume_rate`, `sink_write_rate`, `sink_write_failure_rate`,
43
+ `retry_rate`, `sink_write_latency_ms_avg`, `sink_write_latency_ms_max`.
44
+ Rates are records/second, latencies milliseconds; the window length is
45
+ configurable via the new `MetricsConfig` (`METRICS__WINDOW_SECONDS`,
46
+ default `60`, range 1–600). `IngestGateway(metrics_config=...)` and
47
+ `ConsumerWorker(metrics_config=...)` accept it; when omitted everything
48
+ works with the default window. Purely additive — old consumers that
49
+ deserialize the snapshot ignore the new fields.
50
+ - **`MetricsConfig`** exported from the package root.
51
+
52
+ ### Changed
53
+
54
+ - `ConsumeRuntime.__init__`: the structured-metrics sink parameter
55
+ `metrics` was renamed to `metrics_sink` to free the `metrics` name for
56
+ the new rate-metrics container (`ConsumeMetrics`). No in-repo caller
57
+ passed it explicitly; external callers using `metrics=<MetricsSink>`
58
+ must rename the keyword.
59
+
8
60
  ## [0.3.1] - 2026-09-08
9
61
 
10
62
  ### Fixed
@@ -184,5 +236,7 @@ First public release. Data pipeline framework: conditional admission → reliabl
184
236
  - Mechanisms as core dependencies; all policy carriers (DB drivers, redis, httpx) as optional extras with lazy loading and install-guidance errors.
185
237
  - Typed configuration objects mapping 1:1 to `KAFKA__*` / `CONSUMER__*` / `DB__*` / `REDIS__*` / `BACKPRESSURE__*` environment variables; required settings fail fast at startup.
186
238
 
239
+ [0.5.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.5.0
240
+ [0.4.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.4.0
187
241
  [0.2.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.2.0
188
242
  [0.1.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.1.0
@@ -9,6 +9,7 @@ underscore):
9
9
  | `KAFKA__` | `KafkaConfig` |
10
10
  | `CONSUMER__` | `ConsumerConfig` |
11
11
  | `BACKPRESSURE__` | `BackpressureConfig` |
12
+ | `METRICS__` | `MetricsConfig` |
12
13
 
13
14
  **Fail-fast semantics:** required settings are not defaulted. If a required value
14
15
  is missing, assembly fails at startup with an error that names the setting and
@@ -123,3 +124,16 @@ switch). For dynamic probing inject your own `BackpressureSignal` —
123
124
  | `BACKPRESSURE__FAIL_CLOSED_ON_UNREACHABLE` | `true` | Treat unreachable probe (retries exhausted) as backlog-exceeded. |
124
125
  | `BACKPRESSURE__REJECT_ON_ANY_DEGRADED` | `false` | Legacy escape hatch: reject when any component is degraded. |
125
126
  | `BACKPRESSURE__UNHEALTHY_CHECK_INTERVAL_SECONDS` | `5.0` | Poll period while REJECTING. |
127
+
128
+ ## MetricsConfig (`METRICS__*`)
129
+
130
+ Health-snapshot rate metrics: both `IngestGateway` and `ConsumerWorker`
131
+ maintain in-memory sliding windows and expose computed rates / latencies
132
+ through their health snapshots (`receive_rate`, `sink_write_rate`,
133
+ `produce_latency_ms_avg`, ...). Windowed aggregates are computed on read —
134
+ no background tasks, no extra endpoint; rates decay to `0.0` once traffic
135
+ stops for longer than the window.
136
+
137
+ | Env var | Default | Description |
138
+ |---------|---------|-------------|
139
+ | `METRICS__WINDOW_SECONDS` | `60` | Sliding-window length (seconds) for all rate/latency fields. Valid range 1–600; out-of-range values fail at startup. Short windows react faster but are noisier. |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: streamgate
3
- Version: 0.3.1
3
+ Version: 0.5.0
4
4
  Summary: A pure-core Kafka data pipeline framework: conditional admission, reliable delivery, pluggable sinks via typed hooks.
5
5
  Project-URL: Homepage, https://github.com/pwg-code/streamgate
6
6
  Project-URL: Repository, https://github.com/pwg-code/streamgate
@@ -57,6 +57,7 @@ The wheel installs exactly three dependencies (`aiokafka`, `loguru`, `pydantic`)
57
57
  - **Error classification as a hook.** Transient vs. poison vs. fatal is storage-specific knowledge. Inject an `ErrorClassifier`; the default only knows generic exceptions (and maps unknowns to poison — probe-protected bisect, never silent drops). A production-hardened SQLAlchemy classifier ships as `streamgate.contrib.sql_sink.SQLAlchemyErrorClassifier`.
58
58
  - **Backpressure with hysteresis.** Ingest probes consumer health on a side channel and rejects based on backlog age with separate trip/recover thresholds. The core ships zero-I/O signals (`ManualBackpressureSignal`); the HTTP-probe implementation ships as [`streamgate.contrib.http_probe`](#contrib-official-strategy-implementations).
59
59
  - **DLQ bisection instead of head-of-line blocking.** A poison message never wedges a partition: the failing batch is bisected with probe comparison, the bad record is quarantined to a dead-letter topic, the rest commits.
60
+ - **Ops visibility built into the snapshot.** Health snapshots carry rate/latency metrics (receive rate, admission conflicts, produce success/failure, sink write rate, retries, write latency avg/max) computed over a configurable sliding window — your existing `/health` endpoint doubles as a monitoring feed, no Prometheus required.
60
61
 
61
62
  ## Quick Start
62
63
 
@@ -136,7 +137,7 @@ your HTTP app (presentation is yours: auth/routing/OpenAPI)
136
137
  ▼ │ (bisect + probe compare)
137
138
  ConsumerWorker ──► RecordWriter (your storage) ──────────┘
138
139
  │ └─ ErrorClassifier (injected)
139
- └─ health snapshot (expose it with your own web framework)
140
+ └─ health snapshot (state + rates/latencies; expose it with your own web framework)
140
141
  ```
141
142
 
142
143
  The package contains **zero web-framework code** (no fastapi/uvicorn), and the core layers never import `streamgate.contrib` — enforced by the import-linter layered contract in CI.
@@ -21,6 +21,7 @@ The wheel installs exactly three dependencies (`aiokafka`, `loguru`, `pydantic`)
21
21
  - **Error classification as a hook.** Transient vs. poison vs. fatal is storage-specific knowledge. Inject an `ErrorClassifier`; the default only knows generic exceptions (and maps unknowns to poison — probe-protected bisect, never silent drops). A production-hardened SQLAlchemy classifier ships as `streamgate.contrib.sql_sink.SQLAlchemyErrorClassifier`.
22
22
  - **Backpressure with hysteresis.** Ingest probes consumer health on a side channel and rejects based on backlog age with separate trip/recover thresholds. The core ships zero-I/O signals (`ManualBackpressureSignal`); the HTTP-probe implementation ships as [`streamgate.contrib.http_probe`](#contrib-official-strategy-implementations).
23
23
  - **DLQ bisection instead of head-of-line blocking.** A poison message never wedges a partition: the failing batch is bisected with probe comparison, the bad record is quarantined to a dead-letter topic, the rest commits.
24
+ - **Ops visibility built into the snapshot.** Health snapshots carry rate/latency metrics (receive rate, admission conflicts, produce success/failure, sink write rate, retries, write latency avg/max) computed over a configurable sliding window — your existing `/health` endpoint doubles as a monitoring feed, no Prometheus required.
24
25
 
25
26
  ## Quick Start
26
27
 
@@ -100,7 +101,7 @@ your HTTP app (presentation is yours: auth/routing/OpenAPI)
100
101
  ▼ │ (bisect + probe compare)
101
102
  ConsumerWorker ──► RecordWriter (your storage) ──────────┘
102
103
  │ └─ ErrorClassifier (injected)
103
- └─ health snapshot (expose it with your own web framework)
104
+ └─ health snapshot (state + rates/latencies; expose it with your own web framework)
104
105
  ```
105
106
 
106
107
  The package contains **zero web-framework code** (no fastapi/uvicorn), and the core layers never import `streamgate.contrib` — enforced by the import-linter layered contract in CI.
@@ -26,6 +26,7 @@
26
26
  - **错误分类也是钩子。** "连接超时该重试"还是"数据脏了该隔离",只有懂你存储的人知道。注入一个分类器即可;不注入也有安全的默认行为(不会丢数据)。
27
27
  - **背压带磁滞防抖。** 消费端积压超标自动拒绝接收,回落后自动放行,来回抖动有磁滞带压着。
28
28
  - **坏数据不堵管道。** 一条毒消息不会卡死整个分区:二分定位、隔离进死信 topic,好数据继续走。
29
+ - **运维观测长在快照里。** 健康快照自带滑动窗口算好的速率/延迟指标(接收速率、准入冲突、生产成败、入库速率、重试速率、写入耗时 avg/max),现有健康端点直接当监控数据源用——不配 Prometheus 也能看趋势、设告警。
29
30
 
30
31
  ---
31
32
 
@@ -172,7 +173,11 @@ worker = ConsumerWorker(spec, ..., error_classifier=MyClassifier())
172
173
  ```python
173
174
  class MyAdmission:
174
175
  async def admit(self, record, *, overwrite=False): ... # 判定:允许/冲突/拒绝
175
- async def on_accepted(self, record): ... # Kafka 发送成功后
176
+ async def on_send_success(self, record): ... # 每次 Kafka 发送成功后(通知型)
177
+ async def on_send_failed(self, record): ... # 发送失败后;默认保留占位自愈,
178
+ # 也可在此释放占位换"立即可重发"
179
+ async def on_overwrite_accepted(self, record): ... # 仅 overwrite:摘要写,返回 False
180
+ # → cache_updated=false
176
181
  async def on_persisted(self, record): ... # 消费落库成功后刷新
177
182
  # 另有 start / close / 健康探测方法,见 protocols.py
178
183
 
@@ -294,7 +299,7 @@ pip install "streamgate[http]" # + HTTP 探活背压
294
299
  ▼ │ (二分 + 探针对照)
295
300
  ConsumerWorker ──► RecordWriter(你的存储) ──────────────┘
296
301
  │ └─ ErrorClassifier(注入)
297
- └─ 健康快照(用你自己的 Web 框架暴露)
302
+ └─ 健康快照(状态 + 速率/延迟指标,用你自己的 Web 框架暴露)
298
303
  ```
299
304
 
300
305
  核心层禁止 import `streamgate.contrib`(contrib 反向依赖核心是合法的)——由 import-linter 分层契约在 CI 强制拦截;contrib 的第三方依赖全部走 extras 可选声明,裸装 `pip install streamgate` 不携带。
@@ -22,7 +22,12 @@ Tier 2 逃生口:ConsumeSpec.on_record / ConsumeContext(只读快照)
22
22
  不得深入其余内部子模块(import-linter 门禁强制)。
23
23
  """
24
24
 
25
- from streamgate.config import BackpressureConfig, ConsumerConfig, KafkaConfig
25
+ from streamgate.config import (
26
+ BackpressureConfig,
27
+ ConsumerConfig,
28
+ KafkaConfig,
29
+ MetricsConfig,
30
+ )
26
31
  from streamgate.consumer.classifier import DefaultErrorClassifier
27
32
  from streamgate.consumer.dlq import (
28
33
  BisectOutcome,
@@ -74,7 +79,7 @@ from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT
74
79
  from streamgate.transport.codec import JsonEnvelopeCodec
75
80
  from streamgate.transport.kafka import KafkaConsumerService
76
81
 
77
- __version__ = "0.3.1"
82
+ __version__ = "0.5.0"
78
83
 
79
84
  __all__ = [
80
85
  "AdmissionPolicy",
@@ -113,6 +118,7 @@ __all__ = [
113
118
  "LoggingMetricsSink",
114
119
  "ManualBackpressureSignal",
115
120
  "MessageCodec",
121
+ "MetricsConfig",
116
122
  "MetricsSink",
117
123
  "NoAdmission",
118
124
  "NoBackfill",
@@ -1,87 +1,103 @@
1
- """streamgate 组件配置对象(每组件独立,Spec 内可覆写)。
2
-
3
- 配置键名与既有环境变量逐字对应(KAFKA__* / CONSUMER__* / BACKPRESSURE__*),
4
- 调用方零配置迁移。DB/Redis 是使用方的世界:连接配置由使用方/示例自带
5
- (不进框架核心)。
6
- """
7
-
8
- from pydantic import BaseModel, field_validator
9
-
10
-
11
- class ConsumerConfig(BaseModel):
12
- # 必填:消费服务装配时校验(缺失即启动失败,不做项目专属默认)
13
- group_id: str | None = None
14
- batch_size: int = 100
15
- batch_timeout_seconds: float = 5.0
16
- max_retries: int = 3
17
- retry_backoff_base: float = 1.0
18
- # 重连/暂停恢复指数退避(对齐 ingest producer 的自愈策略,替代原硬编码 1.0/30.0)
19
- reconnect_base_backoff_seconds: float = 1.0
20
- reconnect_max_backoff_seconds: float = 30.0
21
- max_poll_records: int = 500
22
- session_timeout_ms: int = 30000
23
- max_poll_interval_ms: int = 300000
24
- auto_offset_reset: str = "earliest"
25
- backlog_check_interval_seconds: float = 30.0 # 积压时长检查周期
26
- # --- DLQ 隔离:消费端坏数据兜底 ---
27
- dlq_enabled: bool = True # 总开关;false=回退 paused 旧行为(紧急逃生门)
28
- dlq_send_retries: int = 3 # DLQ 单条发送总尝试次数(含首次;耗尽即批次转 paused)
29
-
30
-
31
- class KafkaConfig(BaseModel):
32
- bootstrap_servers: str = "kafka:9092"
33
- # 必填:producer/consumer 装配时校验(缺失即启动失败,不做项目专属默认)
34
- topic: str | None = None
35
- # 死信 topic(消费端隔离坏数据留档);启用 DLQ 时必填
36
- dlq_topic: str | None = None
37
- acks: str = "all"
38
- request_timeout_ms: int = 10000
39
- enable_idempotence: bool = True
40
- # 自愈监控:周期性探活,不健康时销毁旧实例重建连接(覆盖启动期/运行期断线)
41
- health_check_interval_seconds: float = 30.0 # 探活轮询周期
42
- reconnect_base_backoff_seconds: float = 1.0 # 重连指数退避基数
43
- reconnect_max_backoff_seconds: float = 30.0 # 重连退避上限
44
- # R1 防抖:连续 send 失败 N 次才触发重建(默认 1=首次失败即重建;
45
- # broker 秒级抖动时可调高减少误重建)
46
- reconnect_failure_threshold: int = 1
47
- # 探活口径:近窗口内有 send 失败即判定不健康(秒)
48
- send_failure_window_seconds: float = 30.0
49
- # R3 周期分级:异常/重建中高频探活间隔(秒);健康态仍用 health_check_interval_seconds
50
- unhealthy_check_interval_seconds: float = 5.0
51
-
52
- @field_validator("bootstrap_servers", mode="before")
53
- @classmethod
54
- def strip_url_scheme(cls, v: object) -> object:
55
- if not isinstance(v, str):
56
- return v
57
- cleaned: list[str] = []
58
- for part in v.split(","):
59
- part = part.strip()
60
- for scheme in ("http://", "https://", "kafka://"):
61
- if part.lower().startswith(scheme):
62
- part = part[len(scheme):]
63
- break
64
- cleaned.append(part)
65
- return ",".join(cleaned)
66
-
67
-
68
- class BackpressureConfig(BaseModel):
69
- """ingest 背压拒绝配置(consumption-backpressure)。
70
-
71
- 默认值与 existence TTL 联动:trip = existence_ttl/2 = 9000s(漏报窗口打开前拦截,
72
- 与积压告警线 backlog_age_warn 一致);recover = trip*80% = 7200s(磁滞防抖)。
73
- """
74
-
75
- enabled: bool = True # 总开关;False=完全放行(紧急回滚)
76
- consumer_health_url: str = "http://localhost:9109/health" # consumer 健康接口地址
77
- check_interval_seconds: float = 30.0 # 轮询周期(与积压检查周期一致)
78
- timeout_seconds: float = 2.0 # 单次探活超时
79
- probe_retries: int = 3 # 单周期内探活重试次数(不含首次);0=禁用
80
- probe_retry_interval_seconds: float = 15.0 # 重试间隔(秒)
81
- trip_seconds: float = 9000.0 # 触发阈值:积压超此值开始拒绝
82
- recover_seconds: float = 7200.0 # 恢复阈值:积压回落至此值以下放行(磁滞)
83
- retry_after_seconds: int = 60 # 拒绝时 Retry-After 头(秒)
84
- fail_closed_on_unreachable: bool = True # 探活不可达时是否拒绝(fail-closed)
85
- reject_on_any_degraded: bool = False # 旧行为逃生门:任何 degraded 即拒绝(恢复全量拒绝语义)
86
- # R3 背压周期分级:REJECTING 态高频探活间隔(秒);OPEN 态仍用 check_interval_seconds
87
- unhealthy_check_interval_seconds: float = 5.0
1
+ """streamgate 组件配置对象(每组件独立,Spec 内可覆写)。
2
+
3
+ 配置键名与既有环境变量逐字对应(KAFKA__* / CONSUMER__* / BACKPRESSURE__*),
4
+ 调用方零配置迁移。DB/Redis 是使用方的世界:连接配置由使用方/示例自带
5
+ (不进框架核心)。
6
+ """
7
+
8
+ from pydantic import BaseModel, field_validator
9
+
10
+
11
+ class ConsumerConfig(BaseModel):
12
+ # 必填:消费服务装配时校验(缺失即启动失败,不做项目专属默认)
13
+ group_id: str | None = None
14
+ batch_size: int = 100
15
+ batch_timeout_seconds: float = 5.0
16
+ max_retries: int = 3
17
+ retry_backoff_base: float = 1.0
18
+ # 重连/暂停恢复指数退避(对齐 ingest producer 的自愈策略,替代原硬编码 1.0/30.0)
19
+ reconnect_base_backoff_seconds: float = 1.0
20
+ reconnect_max_backoff_seconds: float = 30.0
21
+ max_poll_records: int = 500
22
+ session_timeout_ms: int = 30000
23
+ max_poll_interval_ms: int = 300000
24
+ auto_offset_reset: str = "earliest"
25
+ backlog_check_interval_seconds: float = 30.0 # 积压时长检查周期
26
+ # --- DLQ 隔离:消费端坏数据兜底 ---
27
+ dlq_enabled: bool = True # 总开关;false=回退 paused 旧行为(紧急逃生门)
28
+ dlq_send_retries: int = 3 # DLQ 单条发送总尝试次数(含首次;耗尽即批次转 paused)
29
+
30
+
31
+ class KafkaConfig(BaseModel):
32
+ bootstrap_servers: str = "kafka:9092"
33
+ # 必填:producer/consumer 装配时校验(缺失即启动失败,不做项目专属默认)
34
+ topic: str | None = None
35
+ # 死信 topic(消费端隔离坏数据留档);启用 DLQ 时必填
36
+ dlq_topic: str | None = None
37
+ acks: str = "all"
38
+ request_timeout_ms: int = 10000
39
+ enable_idempotence: bool = True
40
+ # 自愈监控:周期性探活,不健康时销毁旧实例重建连接(覆盖启动期/运行期断线)
41
+ health_check_interval_seconds: float = 30.0 # 探活轮询周期
42
+ reconnect_base_backoff_seconds: float = 1.0 # 重连指数退避基数
43
+ reconnect_max_backoff_seconds: float = 30.0 # 重连退避上限
44
+ # R1 防抖:连续 send 失败 N 次才触发重建(默认 1=首次失败即重建;
45
+ # broker 秒级抖动时可调高减少误重建)
46
+ reconnect_failure_threshold: int = 1
47
+ # 探活口径:近窗口内有 send 失败即判定不健康(秒)
48
+ send_failure_window_seconds: float = 30.0
49
+ # R3 周期分级:异常/重建中高频探活间隔(秒);健康态仍用 health_check_interval_seconds
50
+ unhealthy_check_interval_seconds: float = 5.0
51
+
52
+ @field_validator("bootstrap_servers", mode="before")
53
+ @classmethod
54
+ def strip_url_scheme(cls, v: object) -> object:
55
+ if not isinstance(v, str):
56
+ return v
57
+ cleaned: list[str] = []
58
+ for part in v.split(","):
59
+ part = part.strip()
60
+ for scheme in ("http://", "https://", "kafka://"):
61
+ if part.lower().startswith(scheme):
62
+ part = part[len(scheme):]
63
+ break
64
+ cleaned.append(part)
65
+ return ",".join(cleaned)
66
+
67
+
68
+ class BackpressureConfig(BaseModel):
69
+ """ingest 背压拒绝配置(consumption-backpressure)。
70
+
71
+ 默认值与 existence TTL 联动:trip = existence_ttl/2 = 9000s(漏报窗口打开前拦截,
72
+ 与积压告警线 backlog_age_warn 一致);recover = trip*80% = 7200s(磁滞防抖)。
73
+ """
74
+
75
+ enabled: bool = True # 总开关;False=完全放行(紧急回滚)
76
+ consumer_health_url: str = "http://localhost:9109/health" # consumer 健康接口地址
77
+ check_interval_seconds: float = 30.0 # 轮询周期(与积压检查周期一致)
78
+ timeout_seconds: float = 2.0 # 单次探活超时
79
+ probe_retries: int = 3 # 单周期内探活重试次数(不含首次);0=禁用
80
+ probe_retry_interval_seconds: float = 15.0 # 重试间隔(秒)
81
+ trip_seconds: float = 9000.0 # 触发阈值:积压超此值开始拒绝
82
+ recover_seconds: float = 7200.0 # 恢复阈值:积压回落至此值以下放行(磁滞)
83
+ retry_after_seconds: int = 60 # 拒绝时 Retry-After 头(秒)
84
+ fail_closed_on_unreachable: bool = True # 探活不可达时是否拒绝(fail-closed)
85
+ reject_on_any_degraded: bool = False # 旧行为逃生门:任何 degraded 即拒绝(恢复全量拒绝语义)
86
+ # R3 背压周期分级:REJECTING 态高频探活间隔(秒);OPEN 态仍用 check_interval_seconds
87
+ unhealthy_check_interval_seconds: float = 5.0
88
+
89
+
90
+ class MetricsConfig(BaseModel):
91
+ """健康快照速率指标配置(滑动窗口长度,ingest/consume 两侧共用)。"""
92
+
93
+ window_seconds: int = 60
94
+
95
+ @field_validator("window_seconds")
96
+ @classmethod
97
+ def check_window_seconds(cls, v: int) -> int:
98
+ if not 1 <= v <= 600:
99
+ raise ValueError(
100
+ "window_seconds out of range; "
101
+ "set METRICS__WINDOW_SECONDS to a value between 1 and 600"
102
+ )
103
+ return v