streamgate 0.1.0__tar.gz

streamgate-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+
+# Tooling caches
+.ruff_cache/
+.pyright/
+.mypy_cache/
+.pytest_cache/
+uv.lock.bak
+
+# Environment
+.env
+.env.*
+
+# Local data (sqlite default path) and planning scratch space
+data/
+temp/

streamgate-0.1.0/.import_linter_cache/.gitignore

@@ -0,0 +1,2 @@
+# Automatically created by Grimp.
+*

streamgate-0.1.0/.import_linter_cache/5780e4798d429798e55d962d9844c38a9b5e0494.data.json

@@ -0,0 +1 @@
+{"imports_by_module":{"streamgate":[["streamgate.obs.metrics",false,59,"from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink"],["streamgate.consumer.loop",false,46,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.specs",false,87,"from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT, Upsert"],["streamgate.ingest.producer",false,57,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.protocols",false,60,"from streamgate.protocols import ("],["streamgate.consumer.runner",false,47,"from streamgate.consumer.runner import ConsumerWorker"],["streamgate.resilience.backpressure",false,102,"from streamgate.resilience.backpressure import HttpProbeSignal"],["streamgate.consumer.dlq",false,38,"from streamgate.consumer.dlq import ("],["streamgate.ingest.admission.no_admission",false,55,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.ingest.admission.redis_existence",false,94,"from streamgate.ingest.admission.redis_existence import ("],["streamgate._optional",false,25,"from streamgate._optional import require_optional"],["streamgate.db.backfill",false,48,"from streamgate.db.backfill import SqlBackfill"],["streamgate.obs.logging",false,58,"from streamgate.obs.logging import configure_logger, logger"],["streamgate.db.upsert",false,54,"from streamgate.db.upsert import UpsertWriter"],["streamgate.consumer.classifier",false,33,"from streamgate.consumer.classifier import ("],["streamgate.transport.kafka",false,89,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.db.engines",false,49,"from streamgate.db.engines import ("],["streamgate.resilience.health",false,81,"from streamgate.resilience.health import ("],["streamgate.ingest.gateway",false,56,"from streamgate.ingest.gateway import IngestGateway"],["streamgate.cache.existence",false,93,"from streamgate.cache.existence import EMPTY_FIELD, RedisExistenceCache"],["streamgate.config",false,26,"from streamgate.config import ("],["streamgate.transport.codec",false,88,"from streamgate.transport.codec import JsonEnvelopeCodec"]],"streamgate._optional":[],"streamgate.cache":[],"streamgate.cache.existence":[["streamgate.obs.logging",false,16,"from streamgate.obs.logging import logger"],["streamgate._optional",false,14,"from streamgate._optional import require_optional"],["streamgate.config",false,15,"from streamgate.config import RedisConfig"]],"streamgate.config":[],"streamgate.consumer":[],"streamgate.consumer.classifier":[],"streamgate.consumer.dlq":[["streamgate.obs.logging",false,24,"from streamgate.obs.logging import logger"],["streamgate.protocols",false,25,"from streamgate.protocols import JsonObject, RecordWriter"],["streamgate.config",false,22,"from streamgate.config import KafkaConfig"],["streamgate.consumer.classifier",false,23,"from streamgate.consumer.classifier import FailureCategory"]],"streamgate.consumer.loop":[["streamgate.consumer.dlq",false,18,"from streamgate.consumer.dlq import ("],["streamgate.cache.existence",false,12,"from streamgate.cache.existence import RedisExistenceCache"],["streamgate.obs.logging",false,27,"from streamgate.obs.logging import logger"],["streamgate.consumer.classifier",false,14,"from streamgate.consumer.classifier import ("],["streamgate.protocols",false,29,"from streamgate.protocols import ("],["streamgate.transport.kafka",false,38,"from streamgate.transport.kafka import KafkaConsumerService, KafkaRecord"],["streamgate.specs",false,37,"from streamgate.specs import ConsumeSpec"],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig"],["streamgate.obs.metrics",false,28,"from streamgate.obs.metrics import DEFAULT_METRICS, MetricsSink"]],"streamgate.consumer.runner":[["streamgate.obs.logging",false,23,"from streamgate.obs.logging import logger"],["streamgate.db.upsert",false,22,"from streamgate.db.upsert import UpsertWriter"],["streamgate.db.engines",false,21,"from streamgate.db.engines import async_session_factory, create_write_engine"],["streamgate.config",false,18,"from streamgate.config import ConsumerConfig, DbConfig, KafkaConfig, RedisConfig"],["streamgate.specs",false,29,"from streamgate.specs import ConsumeSpec"],["streamgate.cache.existence",false,17,"from streamgate.cache.existence import RedisExistenceCache"],["streamgate.transport.kafka",false,31,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.protocols",false,24,"from streamgate.protocols import MessageCodec, RecordWriter"],["streamgate.consumer.dlq",false,19,"from streamgate.consumer.dlq import DlqProducer"],["streamgate.consumer.loop",false,20,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.transport.codec",false,30,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.resilience.health",false,25,"from streamgate.resilience.health import ("]],"streamgate.db":[],"streamgate.db.backfill":[["streamgate.protocols",false,10,"from streamgate.protocols import JsonObject"],["streamgate.obs.logging",false,9,"from streamgate.obs.logging import logger"]],"streamgate.db.dialects":[],"streamgate.db.dialects.mssql":[["streamgate.protocols",false,21,"from streamgate.protocols import JsonObject"]],"streamgate.db.dialects.sqlite":[["streamgate.protocols",false,8,"from streamgate.protocols import JsonObject"]],"streamgate.db.engines":[["streamgate.obs.logging",false,12,"from streamgate.obs.logging import logger"],["streamgate.config",false,11,"from streamgate.config import DbConfig"]],"streamgate.db.upsert":[["streamgate.config",false,13,"from streamgate.config import DbConfig"],["streamgate.db.dialects.mssql",false,14,"from streamgate.db.dialects import mssql as mssql_dialect"],["streamgate.protocols",false,17,"from streamgate.protocols import JsonObject, WriteResult"],["streamgate.specs",false,18,"from streamgate.specs import Upsert"],["streamgate.obs.logging",false,16,"from streamgate.obs.logging import logger"],["streamgate.db.dialects.sqlite",false,15,"from streamgate.db.dialects import sqlite as sqlite_dialect"]],"streamgate.ingest":[],"streamgate.ingest.admission":[],"streamgate.ingest.admission.no_admission":[["streamgate.protocols",false,5,"from streamgate.protocols import Decision, RecordT"]],"streamgate.ingest.admission.redis_existence":[["streamgate.cache.existence",false,16,"from streamgate.cache.existence import RedisExistenceCache"],["streamgate.protocols",false,19,"from streamgate.protocols import ("],["streamgate.obs.logging",false,18,"from streamgate.obs.logging import logger"],["streamgate.config",false,17,"from streamgate.config import RedisConfig"]],"streamgate.ingest.gateway":[["streamgate.cache.existence",false,11,"from streamgate.cache.existence import RedisExistenceCache"],["streamgate.protocols",false,20,"from streamgate.protocols import ("],["streamgate.obs.logging",false,19,"from streamgate.obs.logging import logger"],["streamgate.ingest.producer",false,18,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.resilience.health",false,30,"from streamgate.resilience.health import ("],["streamgate.ingest.admission.no_admission",false,13,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.ingest.admission.redis_existence",false,14,"from streamgate.ingest.admission.redis_existence import ("],["streamgate.specs",false,34,"from streamgate.specs import IngestBinding, IngestRecordT"],["streamgate.resilience.backpressure",false,29,"from streamgate.resilience.backpressure import HttpProbeSignal"],["streamgate.config",false,12,"from streamgate.config import BackpressureConfig, DbConfig, KafkaConfig, RedisConfig"],["streamgate.transport.codec",false,35,"from streamgate.transport.codec import JsonEnvelopeCodec"]],"streamgate.ingest.producer":[["streamgate.protocols",false,18,"from streamgate.protocols import JsonObject"],["streamgate.obs.logging",false,17,"from streamgate.obs.logging import logger"],["streamgate.config",false,16,"from streamgate.config import KafkaConfig"]],"streamgate.obs":[],"streamgate.obs.logging":[],"streamgate.obs.metrics":[["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"]],"streamgate.protocols":[],"streamgate.resilience":[],"streamgate.resilience.backpressure":[["streamgate.obs.logging",false,27,"from streamgate.obs.logging import logger"],["streamgate._optional",false,25,"from streamgate._optional import require_optional"],["streamgate.config",false,26,"from streamgate.config import BackpressureConfig"],["streamgate.protocols",false,28,"from streamgate.protocols import BackpressureSnapshot, ProbeResult"]],"streamgate.resilience.health":[["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"]],"streamgate.specs":[["streamgate.protocols",false,18,"from streamgate.protocols import ("],["streamgate.db.dialects.mssql",false,17,"from streamgate.db.dialects.mssql import mssql_cast_types"]],"streamgate.transport":[],"streamgate.transport.codec":[["streamgate.protocols",false,11,"from streamgate.protocols import Envelope, JsonObject"],["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"]],"streamgate.transport.kafka":[["streamgate.obs.logging",false,11,"from streamgate.obs.logging import logger"],["streamgate.config",false,10,"from streamgate.config import ConsumerConfig, KafkaConfig"]]},"version":2}

streamgate-0.1.0/.import_linter_cache/CACHEDIR.TAG

@@ -0,0 +1,3 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag automatically created by Grimp.
+# For information about cache directory tags see https://bford.info/cachedir/

streamgate-0.1.0/.import_linter_cache/streamgate.meta.json

@@ -0,0 +1 @@
+{"streamgate.db.dialects": 1788742270.5761602, "streamgate": 1788776206.1352222, "streamgate.ingest": 1788742270.5761602, "streamgate.config": 1788775744.4866724, "streamgate.transport": 1788742270.5761602, "streamgate._optional": 1788775681.4141757, "streamgate.consumer.runner": 1788767058.740163, "streamgate.db.engines": 1788775754.7772968, "streamgate.db.dialects.sqlite": 1788776206.140225, "streamgate.resilience": 1788742270.5761602, "streamgate.ingest.admission": 1788742270.5761602, "streamgate.db.dialects.mssql": 1788749506.9339142, "streamgate.transport.kafka": 1788776206.140225, "streamgate.db.upsert": 1788766960.2905881, "streamgate.specs": 1788766931.7234573, "streamgate.resilience.health": 1788767769.120218, "streamgate.db.backfill": 1788749357.442897, "streamgate.obs.logging": 1788749262.353655, "streamgate.consumer.classifier": 1788776206.138224, "streamgate.ingest.admission.no_admission": 1788759910.7492158, "streamgate.cache.existence": 1788775707.6089175, "streamgate.consumer.loop": 1788767712.111625, "streamgate.consumer.dlq": 1788750409.4867704, "streamgate.consumer": 1788742270.5761602, "streamgate.ingest.gateway": 1788767162.2992928, "streamgate.transport.codec": 1788749619.0487723, "streamgate.db": 1788742270.5761602, "streamgate.protocols": 1788766898.5526717, "streamgate.obs.metrics": 1788749264.9746833, "streamgate.obs": 1788742270.5761602, "streamgate.cache": 1788742270.5761602, "streamgate.ingest.admission.redis_existence": 1788776250.1590097, "streamgate.ingest.producer": 1788776206.139223, "streamgate.resilience.backpressure": 1788775727.538583}

streamgate-0.1.0/AGENTS.md

@@ -0,0 +1,12 @@
+# AGENTS.md
+
+## 架构与编码规范
+
+**类型与 Pydantic v2**：
+- 所有函数/方法必须标注参数与返回值类型，禁止省略。
+- 禁用 `Any`、裸 `dict` / `list`（无泛型）等无法推断具体结构的类型；类型不确定时用具体类型或 `object` + `isinstance` 守卫。所有结构化数据必须定义对应的 Schema（Pydantic Model / dataclass / TypedDict）。
+- 使用 Python 3.12+ 语法：`A | B`（代替 `Union`）、`list[int]`（代替 `List[int]`）；可能返回空值的函数必须标注 `| None`。
+- 单函数体不超过 50 行，复杂逻辑拆分为带类型注解的子函数；生成代码时先输出数据模型定义，再输出业务逻辑。
+- 禁止手动逐字段映射：用 `model_validate()`（ORM 对象加 `from_attributes=True`）代替 `_to_response()` 和 `Schema(**obj)`；用 `model_dump(exclude=.../exclude_unset=True)` 代替手写字典；字段定义只出现在 Schema 一处。响应体 Schema 有默认值，请求体 Schema 保持必填。
+- 新增代码必须通过 pyright 静态检查。
+

streamgate-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wanguo Pang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

streamgate-0.1.0/PKG-INFO

@@ -0,0 +1,189 @@
+Metadata-Version: 2.5
+Name: streamgate
+Version: 0.1.0
+Summary: A Kafka-backed data pipeline framework: conditional admission, reliable delivery, pluggable sinks.
+Project-URL: Homepage, https://github.com/{github_owner}/streamgate
+Project-URL: Repository, https://github.com/{github_owner}/streamgate
+Project-URL: Changelog, https://github.com/{github_owner}/streamgate/blob/main/CHANGELOG.md
+Author: Wanguo Pang
+License-Expression: MIT
+License-File: LICENSE
+Keywords: backpressure,dlq,ingestion,kafka,pipeline,redis,sqlalchemy
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: aiokafka>=0.10
+Requires-Dist: loguru>=0.7
+Requires-Dist: pydantic>=2.5
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: sqlmodel>=0.0.16
+Provides-Extra: all
+Requires-Dist: aioodbc>=0.5; extra == 'all'
+Requires-Dist: aiosqlite>=0.19; extra == 'all'
+Requires-Dist: httpx>=0.27; extra == 'all'
+Requires-Dist: redis>=5; extra == 'all'
+Provides-Extra: http-probe
+Requires-Dist: httpx>=0.27; extra == 'http-probe'
+Provides-Extra: mssql
+Requires-Dist: aioodbc>=0.5; extra == 'mssql'
+Provides-Extra: redis
+Requires-Dist: redis>=5; extra == 'redis'
+Provides-Extra: sqlite
+Requires-Dist: aiosqlite>=0.19; extra == 'sqlite'
+Description-Content-Type: text/markdown
+
+# streamgate
+
+[![PyPI version](https://img.shields.io/pypi/v/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![Python versions](https://img.shields.io/pypi/pyversions/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/{github_owner}/streamgate/blob/main/LICENSE)
+[![CI](https://github.com/{github_owner}/streamgate/actions/workflows/ci.yml/badge.svg)](https://github.com/{github_owner}/streamgate/actions/workflows/ci.yml)
+
+**streamgate** is a Kafka-backed data pipeline framework: conditional admission at the door, reliable delivery through Kafka, and idempotent sinking wherever you want the data to land.
+
+- 中文提示：本仓库对外文档以英文为主；内部代码注释保留中文。
+
+## Why streamgate
+
+- **Mechanisms in the framework, policies in your hands.** The framework owns admission plumbing, backpressure, health probing, reconnection, offset handling and graceful shutdown. You own entity/slot semantics, schemas and sink targets.
+- **A uniqueness contract, not just "send to Kafka".** Existence checks + atomic slot reservation (Lua, cluster-safe) + cold-path backfill from your database turn at-least-once ingestion into at-most-once admission (409 conflicts instead of duplicates).
+- **Backpressure with hysteresis.** Ingest probes consumer health on a side channel and starts/stop rejecting based on backlog age, with separate trip/recover thresholds so flapping states settle.
+- **DLQ bisection instead of head-of-line blocking.** A poison message never wedges a partition: the failing batch is bisected, the bad record is quarantined to a dead-letter topic, the rest commits.
+- **Tier 0 declarative API covers ~90% of use cases.** Declare what you ingest and where it lands; drop to protocols and escape hatches only when you must.
+
+## Quick Start
+
+```python
+import asyncio
+from pydantic import BaseModel
+from sqlmodel import SQLModel, Field
+
+from streamgate import (
+    BackpressureConfig, ConsumeSpec, ConsumerConfig, ConsumerWorker,
+    DbConfig, IngestBinding, IngestGateway, KafkaConfig, Upsert,
+)
+
+KAFKA = KafkaConfig(bootstrap_servers="localhost:9092", topic="orders")
+
+class OrderIn(BaseModel):          # ingress schema (policy lives with you)
+    order_id: str
+    amount: float
+
+class Order(SQLModel, table=True): # persistence model
+    order_id: str = Field(primary_key=True)
+    amount: float
+
+binding = IngestBinding(
+    message_type="order",
+    entity_key=lambda r: r.order_id,
+    slot_key=lambda r: "order",
+    summary=lambda r: {"amount": r.amount},
+    admission="none",              # or "redis-existence" (extras: streamgate[redis])
+)
+
+async def ingest_one(record: OrderIn) -> None:
+    gateway = IngestGateway(binding=binding, kafka_config=KAFKA,
+                            backpressure_config=BackpressureConfig(enabled=False))
+    await gateway.start()
+    outcome = await gateway.process(record)
+    await gateway.close()
+    print(outcome.kind)
+
+async def consume() -> None:
+    spec = ConsumeSpec(upserts=[Upsert(model=Order, keys=["order_id"])])
+    worker = ConsumerWorker(
+        spec, kafka_config=KAFKA,
+        consumer_config=ConsumerConfig(group_id="order-sink"),
+        db_config=DbConfig(connection_string="sqlite+aiosqlite:///./data/streamgate.db"),
+    )
+    await worker.run()             # blocks until SIGINT/SIGTERM
+```
+
+A runnable version of this (producer + consumer scripts, docker-compose included) lives in [`examples/`](examples/).
+
+## Core concepts
+
+| Tier | What you use | When |
+|------|--------------|------|
+| **0 — Declarative** | `IngestBinding` + `ConsumeSpec`/`Upsert` | ~90% of cases: declare message type, uniqueness keys, sink target |
+| **1 — Component swap** | Protocols in `streamgate.protocols` + built-ins | Replace admission, sinks, backpressure, codec, backfill with your own |
+| **2 — Escape hatch** | `ConsumeSpec.on_record` / `ConsumeContext` | Handle records yourself; no sink machinery at all |
+
+Key protocols (all in `streamgate.protocols`):
+
+- `AdmissionPolicy` — decide admit / 409-conflict / reject per record before it reaches Kafka.
+- `RecordWriter` — where consumed batches land (database, search index, another service...).
+- `BackpressureSignal` — tells the gateway whether to accept; the built-in HTTP probe is just one implementation.
+
+**Stability:** v0.x, experimental. The protocols module is the frozen contract; everything else may still shift.
+
+## Architecture
+
+```
+your HTTP app (presentation is yours: auth/routing/OpenAPI)
+        │
+        ▼
+  IngestGateway ──► backpressure signal (side channel) ─┐
+        │          admission: existence check +         │
+        │          atomic slot reservation (+DB         │
+        │          backfill on cold entities)           │
+        ▼                                               │
+     Kafka ─────────────────────────────────────────────┘  DLQ ◄─ poisoned records
+        │                                                      (bisection)
+        ▼
+  ConsumerWorker ──► sink / upserts / on_record ──► your storage
+        │
+        └─ health snapshot (expose it with your own web framework)
+```
+
+The package contains **zero web-framework code** (no fastapi/uvicorn): HTTP presentation belongs to your adapter layer.
+
+## Installation
+
+```bash
+pip install streamgate              # pure Kafka pipeline: no DB, no redis
+```
+
+Mechanisms are core; policy carriers are extras:
+
+| Extra | Installs | Enables |
+|-------|----------|---------|
+| `[sqlite]` | aiosqlite | built-in upserts / backfill against SQLite |
+| `[mssql]` | aioodbc | built-in upserts / backfill against SQL Server |
+| `[redis]` | redis | `redis-existence` admission strategy |
+| `[http-probe]` | httpx | built-in HTTP backpressure probe signal |
+| `[all]` | all of the above | everything built-in |
+
+```bash
+pip install "streamgate[sqlite]"    # built-in persistence (Quick Start above)
+pip install "streamgate[all]"       # full built-in capability
+```
+
+Other databases (PostgreSQL, MySQL, ...): inject your own `RecordWriter` (Tier 1). The built-in upsert sugar targets SQLite and SQL Server semantics.
+
+## Configuration
+
+All components configure via typed objects that map 1:1 to environment variables (`KAFKA__*`, `CONSUMER__*`, `DB__*`, `REDIS__*`, `BACKPRESSURE__*`). Required settings (e.g. the Kafka topic, consumer group id, and the DB connection string when upserts are declared) fail fast at startup with fix instructions. See [CONFIGURATION.md](CONFIGURATION.md).
+
+## Examples
+
+See [`examples/`](examples/) for a complete ingest → consume → SQLite round trip with a `docker-compose.yml` for Kafka + Redis.
+
+## Roadmap
+
+- Test suite (first release ships without tests; APIs are exercised in production but the project considers this its top debt)
+- Documentation site
+- More admission policies and sink writers
+
+## License
+
+[MIT](LICENSE)

streamgate-0.1.0/README.md

@@ -0,0 +1,146 @@
+# streamgate
+
+[![PyPI version](https://img.shields.io/pypi/v/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![Python versions](https://img.shields.io/pypi/pyversions/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/{github_owner}/streamgate/blob/main/LICENSE)
+[![CI](https://github.com/{github_owner}/streamgate/actions/workflows/ci.yml/badge.svg)](https://github.com/{github_owner}/streamgate/actions/workflows/ci.yml)
+
+**streamgate** is a Kafka-backed data pipeline framework: conditional admission at the door, reliable delivery through Kafka, and idempotent sinking wherever you want the data to land.
+
+- 中文提示：本仓库对外文档以英文为主；内部代码注释保留中文。
+
+## Why streamgate
+
+- **Mechanisms in the framework, policies in your hands.** The framework owns admission plumbing, backpressure, health probing, reconnection, offset handling and graceful shutdown. You own entity/slot semantics, schemas and sink targets.
+- **A uniqueness contract, not just "send to Kafka".** Existence checks + atomic slot reservation (Lua, cluster-safe) + cold-path backfill from your database turn at-least-once ingestion into at-most-once admission (409 conflicts instead of duplicates).
+- **Backpressure with hysteresis.** Ingest probes consumer health on a side channel and starts/stop rejecting based on backlog age, with separate trip/recover thresholds so flapping states settle.
+- **DLQ bisection instead of head-of-line blocking.** A poison message never wedges a partition: the failing batch is bisected, the bad record is quarantined to a dead-letter topic, the rest commits.
+- **Tier 0 declarative API covers ~90% of use cases.** Declare what you ingest and where it lands; drop to protocols and escape hatches only when you must.
+
+## Quick Start
+
+```python
+import asyncio
+from pydantic import BaseModel
+from sqlmodel import SQLModel, Field
+
+from streamgate import (
+    BackpressureConfig, ConsumeSpec, ConsumerConfig, ConsumerWorker,
+    DbConfig, IngestBinding, IngestGateway, KafkaConfig, Upsert,
+)
+
+KAFKA = KafkaConfig(bootstrap_servers="localhost:9092", topic="orders")
+
+class OrderIn(BaseModel):          # ingress schema (policy lives with you)
+    order_id: str
+    amount: float
+
+class Order(SQLModel, table=True): # persistence model
+    order_id: str = Field(primary_key=True)
+    amount: float
+
+binding = IngestBinding(
+    message_type="order",
+    entity_key=lambda r: r.order_id,
+    slot_key=lambda r: "order",
+    summary=lambda r: {"amount": r.amount},
+    admission="none",              # or "redis-existence" (extras: streamgate[redis])
+)
+
+async def ingest_one(record: OrderIn) -> None:
+    gateway = IngestGateway(binding=binding, kafka_config=KAFKA,
+                            backpressure_config=BackpressureConfig(enabled=False))
+    await gateway.start()
+    outcome = await gateway.process(record)
+    await gateway.close()
+    print(outcome.kind)
+
+async def consume() -> None:
+    spec = ConsumeSpec(upserts=[Upsert(model=Order, keys=["order_id"])])
+    worker = ConsumerWorker(
+        spec, kafka_config=KAFKA,
+        consumer_config=ConsumerConfig(group_id="order-sink"),
+        db_config=DbConfig(connection_string="sqlite+aiosqlite:///./data/streamgate.db"),
+    )
+    await worker.run()             # blocks until SIGINT/SIGTERM
+```
+
+A runnable version of this (producer + consumer scripts, docker-compose included) lives in [`examples/`](examples/).
+
+## Core concepts
+
+| Tier | What you use | When |
+|------|--------------|------|
+| **0 — Declarative** | `IngestBinding` + `ConsumeSpec`/`Upsert` | ~90% of cases: declare message type, uniqueness keys, sink target |
+| **1 — Component swap** | Protocols in `streamgate.protocols` + built-ins | Replace admission, sinks, backpressure, codec, backfill with your own |
+| **2 — Escape hatch** | `ConsumeSpec.on_record` / `ConsumeContext` | Handle records yourself; no sink machinery at all |
+
+Key protocols (all in `streamgate.protocols`):
+
+- `AdmissionPolicy` — decide admit / 409-conflict / reject per record before it reaches Kafka.
+- `RecordWriter` — where consumed batches land (database, search index, another service...).
+- `BackpressureSignal` — tells the gateway whether to accept; the built-in HTTP probe is just one implementation.
+
+**Stability:** v0.x, experimental. The protocols module is the frozen contract; everything else may still shift.
+
+## Architecture
+
+```
+your HTTP app (presentation is yours: auth/routing/OpenAPI)
+        │
+        ▼
+  IngestGateway ──► backpressure signal (side channel) ─┐
+        │          admission: existence check +         │
+        │          atomic slot reservation (+DB         │
+        │          backfill on cold entities)           │
+        ▼                                               │
+     Kafka ─────────────────────────────────────────────┘  DLQ ◄─ poisoned records
+        │                                                      (bisection)
+        ▼
+  ConsumerWorker ──► sink / upserts / on_record ──► your storage
+        │
+        └─ health snapshot (expose it with your own web framework)
+```
+
+The package contains **zero web-framework code** (no fastapi/uvicorn): HTTP presentation belongs to your adapter layer.
+
+## Installation
+
+```bash
+pip install streamgate              # pure Kafka pipeline: no DB, no redis
+```
+
+Mechanisms are core; policy carriers are extras:
+
+| Extra | Installs | Enables |
+|-------|----------|---------|
+| `[sqlite]` | aiosqlite | built-in upserts / backfill against SQLite |
+| `[mssql]` | aioodbc | built-in upserts / backfill against SQL Server |
+| `[redis]` | redis | `redis-existence` admission strategy |
+| `[http-probe]` | httpx | built-in HTTP backpressure probe signal |
+| `[all]` | all of the above | everything built-in |
+
+```bash
+pip install "streamgate[sqlite]"    # built-in persistence (Quick Start above)
+pip install "streamgate[all]"       # full built-in capability
+```
+
+Other databases (PostgreSQL, MySQL, ...): inject your own `RecordWriter` (Tier 1). The built-in upsert sugar targets SQLite and SQL Server semantics.
+
+## Configuration
+
+All components configure via typed objects that map 1:1 to environment variables (`KAFKA__*`, `CONSUMER__*`, `DB__*`, `REDIS__*`, `BACKPRESSURE__*`). Required settings (e.g. the Kafka topic, consumer group id, and the DB connection string when upserts are declared) fail fast at startup with fix instructions. See [CONFIGURATION.md](CONFIGURATION.md).
+
+## Examples
+
+See [`examples/`](examples/) for a complete ingest → consume → SQLite round trip with a `docker-compose.yml` for Kafka + Redis.
+
+## Roadmap
+
+- Test suite (first release ships without tests; APIs are exercised in production but the project considers this its top debt)
+- Documentation site
+- More admission policies and sink writers
+
+## License
+
+[MIT](LICENSE)

streamgate-0.1.0/pyproject.toml

@@ -0,0 +1,111 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "streamgate"
+dynamic = ["version"]
+description = "A Kafka-backed data pipeline framework: conditional admission, reliable delivery, pluggable sinks."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "Wanguo Pang" }]
+keywords = [
+    "kafka",
+    "pipeline",
+    "ingestion",
+    "backpressure",
+    "dlq",
+    "sqlalchemy",
+    "redis",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: System :: Distributed Computing",
+    "Typing :: Typed",
+]
+# 核心依赖只含"机制"：传输（aiokafka）、观测（loguru）、配置（pydantic）、
+# Upsert sugar 载体（sqlalchemy + sqlmodel）。
+# 数据库驱动 / redis / httpx 全部为 extras（策略实现按需安装）。
+dependencies = [
+    "aiokafka>=0.10",
+    "loguru>=0.7",
+    "pydantic>=2.5",
+    "sqlalchemy>=2.0",
+    "sqlmodel>=0.0.16",
+]
+
+[project.optional-dependencies]
+sqlite = ["aiosqlite>=0.19"]
+mssql = ["aioodbc>=0.5"]
+redis = ["redis>=5"]
+http-probe = ["httpx>=0.27"]
+all = [
+    "aiosqlite>=0.19",
+    "aioodbc>=0.5",
+    "httpx>=0.27",
+    "redis>=5",
+]
+
+[project.urls]
+Homepage = "https://github.com/{github_owner}/streamgate"
+Repository = "https://github.com/{github_owner}/streamgate"
+Changelog = "https://github.com/{github_owner}/streamgate/blob/main/CHANGELOG.md"
+
+[tool.hatch.version]
+path = "src/streamgate/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/streamgate"]
+
+[dependency-groups]
+dev = [
+    # 可选 extras 的类型来源：TYPE_CHECKING 导入需要 pyright 可解析（CI 类型检查用；
+    # bare-install 冒烟在独立 venv 中进行，不受影响）
+    "aiosqlite>=0.19",
+    "httpx>=0.27",
+    "redis>=5",
+    "import-linter>=2.0",
+    "pyright>=1.1.380",
+    "ruff>=0.6",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+# 显式收敛规则集（CI 确定性）：正确性（E4/E7/E9/F）+ import 排序（I）。
+# 自愈式代码大量使用 except Exception 兜底并记录日志（BLE/S 系不适用）。
+select = ["E4", "E7", "E9", "F", "I"]
+
+[tool.importlinter]
+root_packages = ["streamgate"]
+
+[[tool.importlinter.contracts]]
+name = "Layered architecture (higher layers may import lower layers)"
+type = "layers"
+layers = [
+    "streamgate.ingest | streamgate.consumer",
+    "streamgate.db",
+    "streamgate.cache | streamgate.specs | streamgate.transport | streamgate.resilience",
+    "streamgate.protocols | streamgate.config | streamgate.obs",
+]
+# Tier 0 sugar 载体例外：specs 依赖 mssql 方言的 cast 类型表，
+# 与 db→specs 构成容器级环，此处显式豁免该条导入（真实架构见仓库 README）。
+ignore_imports = [
+    "streamgate.specs -> streamgate.db.dialects.mssql",
+]
+
+[tool.pyright]
+pythonVersion = "3.10"
+include = ["src/streamgate"]
+typeCheckingMode = "basic"