streamgate 0.1.0__tar.gz → 0.2.0__tar.gz

streamgate-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,133 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  checks:
+    name: checks (py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install project and dev tools
+        run: uv sync
+      - name: Lint
+        run: uv run ruff check .
+      - name: Import contracts (layering + pure-core forbidden modules)
+        run: uv run lint-imports
+      - name: Type check
+        run: uv run pyright
+      - name: Build distributions
+        run: uv build
+
+  examples:
+    # Examples are the storefront: they must meet the same quality bar as the
+    # package (lint + typecheck). Dev group carries the third-party libs the
+    # example modules import (sqlalchemy/sqlmodel/redis/httpx/aioodbc).
+    name: examples (lint + typecheck)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - name: Install project and dev tools
+        run: uv sync
+      - name: Lint examples
+        run: uv run ruff check examples
+      - name: Type check examples
+        run: uv run pyright examples
+
+  bare-install:
+    # Regression gate for the pure-core contract: a bare install
+    # (`pip install .`) must pull exactly aiokafka/loguru/pydantic (+transitive),
+    # import cleanly, and run the pure_pipeline example end to end
+    # (ingest → Kafka → consume → sqlite sink) against a real broker.
+    runs-on: ubuntu-latest
+    services:
+      kafka:
+        image: apache/kafka:3.9.0
+        env:
+          KAFKA_NODE_ID: 1
+          KAFKA_PROCESS_ROLES: broker,controller
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+          KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,PLAINTEXT_HOST://:29092
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+          KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+        ports:
+          - 29092:29092
+        options: >-
+          --health-cmd "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092"
+          --health-interval 10s
+          --health-timeout 10s
+          --health-retries 12
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - name: Bare install (no extras — the extras concept no longer exists)
+        run: |
+          uv venv bare
+          uv pip install --python bare/bin/python .
+      - name: "Smoke: dependency tree is pure (no DB/redis/httpx carriers)"
+        run: |
+          bare/bin/python -m pip list --format=freeze > deps.txt
+          cat deps.txt
+          for pkg in sqlalchemy sqlmodel redis aiosqlite aioodbc httpx; do
+            if grep -qi "^${pkg}==" deps.txt; then
+              echo "pure-core violation: ${pkg} leaked into bare install"; exit 1
+            fi
+          done
+      - name: "Smoke: bare import + core contracts"
+        run: |
+          bare/bin/python - <<'EOF'
+          import streamgate
+
+          assert streamgate.__version__
+
+          from streamgate import (
+              ConsumeSpec, DefaultErrorClassifier, ErrorKind,
+              InMemoryAdmission, IngestBinding, IngestGateway,
+              ManualBackpressureSignal,
+          )
+
+          try:
+              ConsumeSpec()
+          except ValueError as e:
+              assert "sink" in str(e) and "on_record" in str(e)
+
+          print("bare-install import smoke OK")
+          EOF
+      - name: "Smoke: pure_pipeline end to end (produce → Kafka → consume → sqlite)"
+        working-directory: examples/pure_pipeline
+        run: |
+          KAFKA__BOOTSTRAP_SERVERS=localhost:29092 ../../../bare/bin/python consume.py &
+          CONSUMER_PID=$!
+          sleep 15
+          KAFKA__BOOTSTRAP_SERVERS=localhost:29092 ../../../bare/bin/python produce.py | tee produce.out
+          grep -q "o-1: accepted" produce.out
+          grep -q "o-1: conflict" produce.out   # in-memory uniqueness contract
+          grep -q "o-2: accepted" produce.out
+          sleep 20   # allow batch-timeout flush + offset commit
+          kill -INT $CONSUMER_PID
+          wait $CONSUMER_PID
+          ../../../bare/bin/python - <<'EOF'
+          import sqlite3
+
+          con = sqlite3.connect("data/pipeline.db")
+          (count,) = con.execute("select count(*) from orders").fetchone()
+          assert count == 2, f"expected 2 orders in sqlite sink, got {count}"
+          print(f"pure_pipeline smoke OK: {count} orders landed in sqlite (o-1 deduplicated)")
+          EOF

streamgate-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: Release
+
+# Publishes to PyPI via Trusted Publisher (OIDC) — no API token stored in the repo.
+#
+# One-time manual setup (outside this repository):
+#   1. Create the PyPI project "streamgate" owner account with 2FA enabled.
+#   2. On PyPI: project -> Publishing -> add a "GitHub" Trusted Publisher:
+#        owner / repository = <owner>/streamgate
+#        workflow filename  = release.yml
+#        environment name   = pypi
+#      (create the protected GitHub environment "pypi" on the repo as well)
+#   3. Rehearse on TestPyPI first by adding an equivalent Trusted Publisher
+#      for testpypi and temporarily retargeting the publish step below.
+#
+# Before tagging, ensure src/streamgate/__init__.py __version__ and CHANGELOG.md
+# both mention the version being released (workflow fails fast on mismatch).
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - name: Verify tag matches package version
+        run: |
+          PKG_VERSION="$(grep -oP '(?<=__version__ = ")[^"]+' src/streamgate/__init__.py)"
+          if [ "v${PKG_VERSION}" != "${GITHUB_REF_NAME}" ]; then
+            echo "tag ${GITHUB_REF_NAME} does not match package version v${PKG_VERSION}"
+            exit 1
+          fi
+      - name: Build distributions
+        run: uv build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

streamgate-0.2.0/.import_linter_cache/5780e4798d429798e55d962d9844c38a9b5e0494.data.json

@@ -0,0 +1 @@
+{"imports_by_module":{"streamgate":[["streamgate.consumer.dlq",false,26,"from streamgate.consumer.dlq import ("],["streamgate.consumer.runner",false,35,"from streamgate.consumer.runner import ConsumerWorker"],["streamgate.specs",false,72,"from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT"],["streamgate.consumer.loop",false,34,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.consumer.classifier",false,25,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["streamgate.ingest.admission.no_admission",false,37,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.ingest.producer",false,39,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.transport.codec",false,73,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.resilience.health",false,66,"from streamgate.resilience.health import ("],["streamgate.obs.logging",false,40,"from streamgate.obs.logging import configure_logger, logger"],["streamgate.protocols",false,42,"from streamgate.protocols import ("],["streamgate.ingest.admission.in_memory",false,36,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.config",false,24,"from streamgate.config import BackpressureConfig, ConsumerConfig, KafkaConfig"],["streamgate.obs.metrics",false,41,"from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink"],["streamgate.resilience.backpressure",false,65,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.transport.kafka",false,74,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.ingest.gateway",false,38,"from streamgate.ingest.gateway import IngestGateway"]],"streamgate.config":[],"streamgate.consumer":[],"streamgate.consumer.classifier":[["streamgate.protocols",false,20,"from streamgate.protocols import ErrorKind"]],"streamgate.consumer.dlq":[["streamgate.config",false,22,"from streamgate.config import KafkaConfig"],["streamgate.obs.logging",false,23,"from streamgate.obs.logging import logger"],["streamgate.protocols",false,24,"from streamgate.protocols import ErrorKind, JsonObject, RecordWriter"]],"streamgate.consumer.loop":[["streamgate.consumer.classifier",false,14,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["streamgate.consumer.dlq",false,15,"from streamgate.consumer.dlq import ("],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig"],["streamgate.obs.metrics",false,25,"from streamgate.obs.metrics import DEFAULT_METRICS, MetricsSink"],["streamgate.protocols",false,26,"from streamgate.protocols import ("],["streamgate.specs",false,36,"from streamgate.specs import ConsumeSpec"],["streamgate.transport.kafka",false,37,"from streamgate.transport.kafka import KafkaConsumerService, KafkaRecord"],["streamgate.obs.logging",false,24,"from streamgate.obs.logging import logger"]],"streamgate.consumer.runner":[["streamgate.protocols",false,15,"from streamgate.protocols import ErrorClassifier, MessageCodec"],["streamgate.config",false,11,"from streamgate.config import ConsumerConfig, KafkaConfig"],["streamgate.consumer.dlq",false,12,"from streamgate.consumer.dlq import DlqProducer"],["streamgate.specs",false,20,"from streamgate.specs import ConsumeSpec"],["streamgate.obs.logging",false,14,"from streamgate.obs.logging import logger"],["streamgate.resilience.health",false,16,"from streamgate.resilience.health import ("],["streamgate.transport.codec",false,21,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.transport.kafka",false,22,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.consumer.loop",false,13,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"]],"streamgate.ingest":[],"streamgate.ingest.admission":[],"streamgate.ingest.admission.in_memory":[["streamgate.protocols",false,15,"from streamgate.protocols import Decision, JsonObject, RecordT"]],"streamgate.ingest.admission.no_admission":[["streamgate.protocols",false,5,"from streamgate.protocols import Decision, RecordT"]],"streamgate.ingest.gateway":[["streamgate.config",false,11,"from streamgate.config import BackpressureConfig, KafkaConfig"],["streamgate.protocols",false,16,"from streamgate.protocols import ("],["streamgate.transport.codec",false,31,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.ingest.producer",false,14,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.resilience.backpressure",false,25,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"],["streamgate.ingest.admission.no_admission",false,13,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.resilience.health",false,26,"from streamgate.resilience.health import ("],["streamgate.ingest.admission.in_memory",false,12,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.specs",false,30,"from streamgate.specs import IngestBinding, IngestRecordT"]],"streamgate.ingest.producer":[["streamgate.config",false,16,"from streamgate.config import KafkaConfig"],["streamgate.protocols",false,18,"from streamgate.protocols import JsonObject"],["streamgate.obs.logging",false,17,"from streamgate.obs.logging import logger"]],"streamgate.obs":[],"streamgate.obs.logging":[],"streamgate.obs.metrics":[["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"]],"streamgate.protocols":[],"streamgate.resilience":[],"streamgate.resilience.backpressure":[["streamgate.protocols",false,10,"from streamgate.protocols import AllowAllSignal, BackpressureSnapshot"]],"streamgate.resilience.health":[["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"]],"streamgate.specs":[["streamgate.protocols",false,16,"from streamgate.protocols import ("]],"streamgate.transport":[],"streamgate.transport.codec":[["streamgate.protocols",false,11,"from streamgate.protocols import Envelope, JsonObject"],["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"]],"streamgate.transport.kafka":[["streamgate.obs.logging",false,11,"from streamgate.obs.logging import logger"],["streamgate.config",false,10,"from streamgate.config import ConsumerConfig, KafkaConfig"]]},"version":2}

streamgate-0.2.0/.import_linter_cache/a1215eee6d5c7b9f11de3767d6965c4a9c703b3e.data.json

@@ -0,0 +1 @@
+{"imports_by_module":{"streamgate":[["streamgate.ingest.gateway",false,38,"from streamgate.ingest.gateway import IngestGateway"],["streamgate.ingest.admission.in_memory",false,36,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.consumer.classifier",false,25,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["streamgate.consumer.loop",false,34,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.consumer.dlq",false,26,"from streamgate.consumer.dlq import ("],["streamgate.specs",false,72,"from streamgate.specs import ConsumeSpec, IngestBinding, IngestRecordT"],["streamgate.config",false,24,"from streamgate.config import BackpressureConfig, ConsumerConfig, KafkaConfig"],["streamgate.resilience.backpressure",false,65,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.resilience.health",false,66,"from streamgate.resilience.health import ("],["streamgate.protocols",false,42,"from streamgate.protocols import ("],["streamgate.ingest.producer",false,39,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.obs.metrics",false,41,"from streamgate.obs.metrics import LoggingMetricsSink, MetricsSink"],["streamgate.transport.kafka",false,74,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.ingest.admission.no_admission",false,37,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.obs.logging",false,40,"from streamgate.obs.logging import configure_logger, logger"],["streamgate.transport.codec",false,73,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.consumer.runner",false,35,"from streamgate.consumer.runner import ConsumerWorker"]],"streamgate.config":[["pydantic",false,8,"from pydantic import BaseModel, field_validator"]],"streamgate.consumer":[],"streamgate.consumer.classifier":[["streamgate.protocols",false,20,"from streamgate.protocols import ErrorKind"],["asyncio",false,16,"import asyncio"],["aiokafka",false,18,"from aiokafka.errors import KafkaError"]],"streamgate.consumer.dlq":[["json",false,16,"import json"],["asyncio",false,15,"import asyncio"],["aiokafka",false,20,"from aiokafka import AIOKafkaProducer"],["streamgate.config",false,22,"from streamgate.config import KafkaConfig"],["streamgate.obs.logging",false,23,"from streamgate.obs.logging import logger"],["datetime",false,18,"from datetime import datetime, timezone"],["dataclasses",false,17,"from dataclasses import dataclass, field"],["streamgate.protocols",false,24,"from streamgate.protocols import ErrorKind, JsonObject, RecordWriter"]],"streamgate.consumer.loop":[["streamgate.specs",false,36,"from streamgate.specs import ConsumeSpec"],["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.config",false,13,"from streamgate.config import ConsumerConfig"],["time",false,6,"import time"],["collections",false,7,"from collections.abc import Callable"],["asyncio",false,5,"import asyncio"],["streamgate.consumer.dlq",false,15,"from streamgate.consumer.dlq import ("],["aiokafka",false,10,"from aiokafka.errors import CommitFailedError"],["streamgate.obs.metrics",false,25,"from streamgate.obs.metrics import DEFAULT_METRICS, MetricsSink"],["streamgate.consumer.classifier",false,14,"from streamgate.consumer.classifier import DefaultErrorClassifier"],["aiokafka",false,11,"from aiokafka.structs import OffsetAndMetadata, TopicPartition"],["streamgate.obs.logging",false,24,"from streamgate.obs.logging import logger"],["streamgate.transport.kafka",false,37,"from streamgate.transport.kafka import KafkaConsumerService, KafkaRecord"],["streamgate.protocols",false,26,"from streamgate.protocols import ("]],"streamgate.consumer.runner":[["streamgate.consumer.loop",false,13,"from streamgate.consumer.loop import ConsumeRuntime, consume_loop"],["streamgate.consumer.dlq",false,12,"from streamgate.consumer.dlq import DlqProducer"],["streamgate.resilience.health",false,16,"from streamgate.resilience.health import ("],["asyncio",false,8,"import asyncio"],["streamgate.config",false,11,"from streamgate.config import ConsumerConfig, KafkaConfig"],["streamgate.obs.logging",false,14,"from streamgate.obs.logging import logger"],["signal",false,9,"import signal"],["streamgate.protocols",false,15,"from streamgate.protocols import ErrorClassifier, MessageCodec"],["streamgate.transport.kafka",false,22,"from streamgate.transport.kafka import KafkaConsumerService"],["streamgate.specs",false,20,"from streamgate.specs import ConsumeSpec"],["streamgate.transport.codec",false,21,"from streamgate.transport.codec import JsonEnvelopeCodec"]],"streamgate.ingest":[],"streamgate.ingest.admission":[],"streamgate.ingest.admission.in_memory":[["typing",false,13,"from typing import Generic"],["streamgate.protocols",false,15,"from streamgate.protocols import Decision, JsonObject, RecordT"],["collections",false,12,"from collections.abc import Callable"]],"streamgate.ingest.admission.no_admission":[["streamgate.protocols",false,5,"from streamgate.protocols import Decision, RecordT"],["typing",false,3,"from typing import Generic"]],"streamgate.ingest.gateway":[["streamgate.ingest.admission.in_memory",false,12,"from streamgate.ingest.admission.in_memory import InMemoryAdmission"],["streamgate.specs",false,30,"from streamgate.specs import IngestBinding, IngestRecordT"],["datetime",false,8,"from datetime import datetime, timezone"],["streamgate.resilience.health",false,26,"from streamgate.resilience.health import ("],["streamgate.resilience.backpressure",false,25,"from streamgate.resilience.backpressure import ManualBackpressureSignal"],["streamgate.ingest.producer",false,14,"from streamgate.ingest.producer import KafkaProducerService"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"],["typing",false,9,"from typing import Generic"],["streamgate.config",false,11,"from streamgate.config import BackpressureConfig, KafkaConfig"],["streamgate.ingest.admission.no_admission",false,13,"from streamgate.ingest.admission.no_admission import NoAdmission"],["streamgate.transport.codec",false,31,"from streamgate.transport.codec import JsonEnvelopeCodec"],["streamgate.protocols",false,16,"from streamgate.protocols import ("]],"streamgate.ingest.producer":[["aiokafka",false,13,"from aiokafka import AIOKafkaProducer"],["datetime",false,11,"from datetime import datetime, timezone"],["aiokafka",false,14,"from aiokafka.errors import KafkaConnectionError, KafkaError"],["streamgate.protocols",false,18,"from streamgate.protocols import JsonObject"],["json",false,9,"import json"],["streamgate.obs.logging",false,17,"from streamgate.obs.logging import logger"],["streamgate.config",false,16,"from streamgate.config import KafkaConfig"],["time",false,10,"import time"],["asyncio",false,8,"import asyncio"]],"streamgate.obs":[],"streamgate.obs.logging":[["loguru",false,6,"from loguru._handler import Message"],["json",false,1,"import json"],["datetime",false,3,"from datetime import timezone"],["sys",false,2,"import sys"],["loguru",false,5,"from loguru import logger as _logger"]],"streamgate.obs.metrics":[["typing",false,8,"from typing import Protocol"],["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"]],"streamgate.protocols":[["collections",false,7,"from collections.abc import Awaitable, Callable"],["datetime",false,9,"from datetime import datetime"],["enum",false,10,"from enum import Enum"],["typing",false,11,"from typing import Protocol, TypeVar, runtime_checkable"],["dataclasses",false,8,"from dataclasses import dataclass, field"]],"streamgate.resilience":[],"streamgate.resilience.backpressure":[["streamgate.protocols",false,10,"from streamgate.protocols import AllowAllSignal, BackpressureSnapshot"]],"streamgate.resilience.health":[["datetime",false,10,"from datetime import datetime, timezone"],["typing",false,11,"from typing import Protocol"],["time",false,8,"import time"],["pydantic",false,13,"from pydantic import BaseModel, Field"],["asyncio",false,7,"import asyncio"],["collections",false,9,"from collections.abc import Awaitable"],["streamgate.obs.logging",false,15,"from streamgate.obs.logging import logger"]],"streamgate.specs":[["dataclasses",false,11,"from dataclasses import dataclass, field"],["collections",false,10,"from collections.abc import Callable, Hashable"],["typing",false,12,"from typing import Generic, Literal, TypeVar"],["pydantic",false,14,"from pydantic import BaseModel"],["streamgate.protocols",false,16,"from streamgate.protocols import ("]],"streamgate.transport":[],"streamgate.transport.codec":[["json",false,8,"import json"],["streamgate.obs.logging",false,10,"from streamgate.obs.logging import logger"],["streamgate.protocols",false,11,"from streamgate.protocols import Envelope, JsonObject"]],"streamgate.transport.kafka":[["aiokafka",false,7,"from aiokafka.errors import CommitFailedError, KafkaConnectionError, KafkaError"],["aiokafka",false,8,"from aiokafka.structs import ConsumerRecord, OffsetAndMetadata, TopicPartition"],["asyncio",false,3,"import asyncio"],["aiokafka",false,5,"from aiokafka import AIOKafkaConsumer"],["aiokafka",false,6,"from aiokafka.consumer.group_coordinator import GroupCoordinator"],["streamgate.obs.logging",false,11,"from streamgate.obs.logging import logger"],["streamgate.config",false,10,"from streamgate.config import ConsumerConfig, KafkaConfig"]]},"version":2}

streamgate-0.2.0/.import_linter_cache/streamgate.meta.json

@@ -0,0 +1 @@
+{"streamgate.ingest.producer": 1788776206.139223, "streamgate.resilience": 1788742270.5761602, "streamgate.resilience.backpressure": 1788778626.931601, "streamgate.obs.metrics": 1788749264.9746833, "streamgate.protocols": 1788780004.1189399, "streamgate.transport.codec": 1788749619.0487723, "streamgate.specs": 1788778652.1194282, "streamgate": 1788778876.7474682, "streamgate.consumer": 1788742270.5761602, "streamgate.resilience.health": 1788767769.120218, "streamgate.obs.logging": 1788749262.353655, "streamgate.consumer.classifier": 1788778605.4753034, "streamgate.transport.kafka": 1788776206.140225, "streamgate.consumer.runner": 1788778819.43872, "streamgate.ingest.gateway": 1788778851.3711455, "streamgate.ingest.admission": 1788742270.5761602, "streamgate.config": 1788778673.809221, "streamgate.obs": 1788742270.5761602, "streamgate.ingest.admission.no_admission": 1788759910.7492158, "streamgate.ingest": 1788742270.5761602, "streamgate.ingest.admission.in_memory": 1788778617.9172873, "streamgate.transport": 1788742270.5761602, "streamgate.consumer.dlq": 1788778795.1676013, "streamgate.consumer.loop": 1788778765.328234}

streamgate-0.2.0/CHANGELOG.md

@@ -0,0 +1,116 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] - 2026-09-07
+
+**Pure-core release.** All database, Redis and HTTP-client code is removed from
+the wheel; every I/O strategy is now a typed protocol hook with a copy-paste
+reference implementation in `examples/`. Core dependencies collapse to
+`aiokafka` + `loguru` + `pydantic`. This is a breaking release (0.1.0 was
+published days earlier; the migration cost is intentionally accepted).
+
+### Changed
+
+- **Core dependencies**: `aiokafka` / `loguru` / `pydantic` only. The
+  `sqlalchemy` and `sqlmodel` dependencies — and the entire extras system
+  (`[sqlite]`, `[mssql]`, `[redis]`, `[http-probe]`, `[all]`) — are gone.
+- **`ConsumeSpec`**: landing targets reduced to two, mutually exclusive:
+  `sink` (a `RecordWriter` — the only persistence path) or `on_record`
+  (escape hatch). Providing neither — or both — fails fast at declaration
+  with fix guidance.
+- **`IngestBinding.admission`**: string shortcuts are now `"none"` and
+  `"in-memory"` (new built-in, zero-I/O, single-process). `"redis-existence"`
+  is removed — inject a `RedisExistenceAdmission` instance instead
+  (copy from `examples/redis_admission/`).
+- **`IngestGateway`**: default backpressure signal is now the built-in
+  `ManualBackpressureSignal` (static switch, defaults to not rejecting);
+  `db_config` / `redis_config` constructor parameters removed.
+- **`ConsumerWorker`**: constructor parameters `db_config`, `redis_config`,
+  `writer`, `engine`, `session_factory` removed — the sink comes from
+  `ConsumeSpec.sink`. New `error_classifier` parameter (see below).
+
+### Added
+
+- **`ErrorClassifier` protocol + `ErrorKind` enum** (`RETRY` / `POISON` /
+  `FATAL`): write-failure classification is now an injected policy.
+  `DefaultErrorClassifier` only recognizes generic exceptions
+  (timeouts / KafkaError → retry; ValueError → poison; unknown → poison,
+  probe-protected bisect). Consumers writing to a database **must** inject a
+  DB-aware classifier — `examples/sqlite_sink/classifier.py` ships a
+  production-hardened `SQLAlchemyErrorClassifier` (incl. SQL Server error
+  numbers).
+- **`ManualBackpressureSignal`**: built-in pure in-memory backpressure switch.
+- **`InMemoryAdmission`**: built-in zero-I/O uniqueness admission
+  (single-process only; multi-instance topologies should inject a
+  shared-storage policy).
+- **`examples/`** (not part of the wheel, CI-gated for lint/typecheck):
+  - `pure_pipeline/` — bare-install full round trip (main example; the
+    living proof of `pip install streamgate` purity)
+  - `sqlite_sink/` — the former `db/` package (engines / upsert sugar /
+    backfill / SQLite+MSSQL dialects) as local modules + `RecordWriter`
+    injection + `SQLAlchemyErrorClassifier`
+  - `redis_admission/` — the former `cache/existence.py` +
+    `ingest/admission/redis_existence.py` + `AdmissionPolicy` injection
+  - `http_probe/` — the former `HttpProbeSignal` (hysteresis state machine) +
+    `BackpressureSignal` injection
+
+### Removed
+
+- `streamgate.db` package (`UpsertWriter`, `SqlBackfill`, engine factories,
+  SQLite/MSSQL dialects), `Upsert` declaration, `DbConfig` / `RedisConfig`
+  (and their `DB__*` / `REDIS__*` env mappings), `RedisExistenceCache`,
+  `RedisExistenceAdmission` (+ related symbols), `HttpProbeSignal`,
+  `_optional` lazy-loading gate.
+- `FailureCategory` / `add_failure_rule` / `classify_write_failure`
+  (superseded by `ErrorClassifier`).
+
+### Migration Guide (0.1.0 → 0.2.0)
+
+| 0.1.0 symbol | Where it went |
+|--------------|---------------|
+| `Upsert` | `examples/sqlite_sink/upsert.py` (`Upsert`, local class) |
+| `UpsertWriter` | `examples/sqlite_sink/upsert.py` — inject via `ConsumeSpec(sink=UpsertWriter(...))` |
+| `DbConfig`, `create_read_engine`, `create_write_engine`, `async_session_factory` | `examples/sqlite_sink/config.py` + `engines.py` |
+| `SqlBackfill` | `examples/sqlite_sink/backfill.py` |
+| SQLite / MSSQL dialect internals | `examples/sqlite_sink/dialects/` |
+| `RedisExistenceCache`, `EMPTY_FIELD` | `examples/redis_admission/existence.py` |
+| `RedisExistenceAdmission`, `RedisExistenceAdmissionConfig`, `EntitySlots`, `SlotSource`, `ExistenceUnavailableError`, `UndeterminedReason` | `examples/redis_admission/admission.py` — inject via `IngestBinding(admission=RedisExistenceAdmission(...))` |
+| `HttpProbeSignal` | `examples/http_probe/probe_signal.py` — inject via `IngestGateway(signal=HttpProbeSignal(...))` |
+| `FailureCategory`, `classify_write_failure`, `add_failure_rule` | `ErrorKind` + `ErrorClassifier`; DB mapping in `examples/sqlite_sink/classifier.py` |
+| `ConsumeSpec(upserts=[...])` | `ConsumeSpec(sink=UpsertWriter(..., [Upsert(...)]))` (writer built by you) |
+| `ConsumerWorker(db_config=..., spec=ConsumeSpec(upserts=...))` | `ConsumerWorker(spec, ...)` with `ConsumeSpec(sink=...)`; pass `error_classifier=` |
+| `admission="redis-existence"` | `admission=RedisExistenceAdmission(...)` (instance injection) |
+| `pip install "streamgate[redis]"` etc. | `pip install streamgate <third-party-lib>` + copy the example module |
+
+## [0.1.0] - 2026-09-07
+
+First public release. Data pipeline framework: conditional admission → reliable delivery → pluggable sinks.
+
+### Added
+
+- **Ingest (admission)**
+  - `IngestGateway`: transport-agnostic ingest kernel — backpressure check → admission → Kafka send → on_accepted orchestration.
+  - `IngestBinding`: declarative ingest binding (Tier 0) with `admission="none" | "redis-existence"` shortcuts or a custom `AdmissionPolicy`.
+  - `RedisExistenceAdmission` / `RedisExistenceCache` (extras: `[redis]`): uniqueness contract via existence check + atomic slot reservation (cluster-safe Lua) + TTL idle-GC + empty-entity sentinels + fail-closed semantics; optional cold-entity DB backfill via `BackfillSource` (`SqlBackfill`).
+- **Transport**
+  - `KafkaProducerService` / `KafkaConsumerService` over aiokafka: lazy connect, health probing, self-healing reconnects with exponential backoff and rebuild thresholds.
+  - `JsonEnvelopeCodec` envelope format; custom `MessageCodec` supported.
+- **Consumer (sink)**
+  - `ConsumerWorker` + `consume_loop`: batch buffering, at-least-once processing, graceful shutdown.
+  - `ConsumeSpec` (Tier 0) with three mutually exclusive landing targets: `upserts` (built-in idempotent SQL upserts via `Upsert`), `sink` (`RecordWriter` injection point), `on_record` (no-sink escape hatch).
+  - Built-in SQLite / SQL Server upsert dialects (extras: `[sqlite]` / `[mssql]`); other databases via Tier 1 `RecordWriter`.
+  - DLQ bisection: poison records are located by batch bisection and quarantined to a dead-letter topic instead of wedging the partition.
+- **Resilience & observability**
+  - Consumption backpressure with hysteresis (trip/recover on backlog age), fail-closed probing, `HttpProbeSignal` (extras: `[http-probe]`); custom `BackpressureSignal` supported.
+  - Health snapshots for both processes (`collect_ingest_health` / `collect_consumer_health`); exposure is left to the caller (the package contains no web-framework code).
+  - Structured logging via loguru (`configure_logger`, `logger`) and a pluggable `MetricsSink`.
+- **Packaging**
+  - Mechanisms as core dependencies; all policy carriers (DB drivers, redis, httpx) as optional extras with lazy loading and install-guidance errors.
+  - Typed configuration objects mapping 1:1 to `KAFKA__*` / `CONSUMER__*` / `DB__*` / `REDIS__*` / `BACKPRESSURE__*` environment variables; required settings fail fast at startup.
+
+[0.2.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.2.0
+[0.1.0]: https://github.com/pwg-code/streamgate/releases/tag/v0.1.0

streamgate-0.2.0/CONFIGURATION.md

@@ -0,0 +1,79 @@
+# Configuration reference
+
+streamgate components are configured with typed Pydantic objects whose fields map
+1:1 to environment variables using a `PREFIX__field` convention (double
+underscore):
+
+| Prefix | Config object |
+|--------|---------------|
+| `KAFKA__` | `KafkaConfig` |
+| `CONSUMER__` | `ConsumerConfig` |
+| `BACKPRESSURE__` | `BackpressureConfig` |
+
+**Fail-fast semantics:** required settings are not defaulted. If a required value
+is missing, assembly fails at startup with an error that names the setting and
+how to fix it — the process never runs with project-specific guessed defaults.
+
+Database and Redis connection settings are **not part of the framework**:
+storage carriers are injected via protocols and their configuration belongs to
+your application (see the example-local configs in `examples/sqlite_sink/` and
+`examples/redis_admission/`).
+
+---
+
+## KafkaConfig (`KAFKA__*`)
+
+| Env var | Default | Description |
+|---------|---------|-------------|
+| `KAFKA__BOOTSTRAP_SERVERS` | `kafka:9092` | Comma-separated broker list. `http://` / `https://` / `kafka://` scheme prefixes are stripped automatically. |
+| `KAFKA__TOPIC` | **required** | Default topic for producer and consumer. Missing ⇒ startup failure (producer/consumer assembly validates). |
+| `KAFKA__DLQ_TOPIC` | `None` (required when DLQ enabled) | Dead-letter topic for quarantined poison records. Required when `dlq_enabled` is on and no per-spec override is set. |
+| `KAFKA__ACKS` | `all` | Producer acks level. |
+| `KAFKA__REQUEST_TIMEOUT_MS` | `10000` | Producer request timeout (ms). |
+| `KAFKA__ENABLE_IDEMPOTENCE` | `true` | Idempotent producer (safe retries). |
+| `KAFKA__HEALTH_CHECK_INTERVAL_SECONDS` | `30.0` | Periodic health probe interval; unhealthy instances are rebuilt. |
+| `KAFKA__RECONNECT_BASE_BACKOFF_SECONDS` | `1.0` | Reconnect exponential backoff base. |
+| `KAFKA__RECONNECT_MAX_BACKOFF_SECONDS` | `30.0` | Reconnect backoff cap. |
+| `KAFKA__RECONNECT_FAILURE_THRESHOLD` | `1` | Consecutive send failures before a rebuild is triggered (raise to tolerate broker jitter). |
+| `KAFKA__SEND_FAILURE_WINDOW_SECONDS` | `30.0` | A send failure within this window marks the instance unhealthy. |
+| `KAFKA__UNHEALTHY_CHECK_INTERVAL_SECONDS` | `5.0` | High-frequency probe interval while unhealthy/rebuilding. |
+
+## ConsumerConfig (`CONSUMER__*`)
+
+| Env var | Default | Description |
+|---------|---------|-------------|
+| `CONSUMER__GROUP_ID` | **required** | Kafka consumer group id. Missing ⇒ startup failure. |
+| `CONSUMER__BATCH_SIZE` | `100` | Records buffered before a write attempt. |
+| `CONSUMER__BATCH_TIMEOUT_SECONDS` | `5.0` | Max wait before a partially filled batch is flushed. |
+| `CONSUMER__MAX_RETRIES` | `3` | Write retries per batch. |
+| `CONSUMER__RETRY_BACKOFF_BASE` | `1.0` | Write retry backoff base (seconds). |
+| `CONSUMER__RECONNECT_BASE_BACKOFF_SECONDS` | `1.0` | Consumer reconnect backoff base. |
+| `CONSUMER__RECONNECT_MAX_BACKOFF_SECONDS` | `30.0` | Consumer reconnect backoff cap. |
+| `CONSUMER__MAX_POLL_RECORDS` | `500` | Max records per poll. |
+| `CONSUMER__SESSION_TIMEOUT_MS` | `30000` | Kafka session timeout. |
+| `CONSUMER__MAX_POLL_INTERVAL_MS` | `300000` | Max time between polls before rebalance. |
+| `CONSUMER__AUTO_OFFSET_RESET` | `earliest` | Offset reset policy. |
+| `CONSUMER__BACKLOG_CHECK_INTERVAL_SECONDS` | `30.0` | Backlog age check period (feeds backpressure metrics). |
+| `CONSUMER__DLQ_ENABLED` | `true` | DLQ master switch; `false` falls back to the legacy paused behavior (emergency escape hatch). |
+| `CONSUMER__DLQ_SEND_RETRIES` | `3` | Total attempts per DLQ send (exhausting them pauses the batch). |
+
+## BackpressureConfig (`BACKPRESSURE__*`)
+
+The gateway defaults to the built-in `ManualBackpressureSignal` (static
+switch). For dynamic probing inject your own `BackpressureSignal` — the
+HTTP-probe reference in `examples/http_probe/` reads this config object.
+
+| Env var | Default | Description |
+|---------|---------|-------------|
+| `BACKPRESSURE__ENABLED` | `true` | Master switch; `false` = fully open (emergency rollback). |
+| `BACKPRESSURE__CONSUMER_HEALTH_URL` | `http://localhost:9109/health` | Consumer health endpoint polled by the probe. |
+| `BACKPRESSURE__CHECK_INTERVAL_SECONDS` | `30.0` | Poll period while OPEN. |
+| `BACKPRESSURE__TIMEOUT_SECONDS` | `2.0` | Per-probe timeout. |
+| `BACKPRESSURE__PROBE_RETRIES` | `3` | Retries per cycle (excluding the first attempt); `0` disables. |
+| `BACKPRESSURE__PROBE_RETRY_INTERVAL_SECONDS` | `15.0` | Interval between retries. |
+| `BACKPRESSURE__TRIP_SECONDS` | `9000.0` | Reject when backlog age exceeds this. |
+| `BACKPRESSURE__RECOVER_SECONDS` | `7200.0` | Resume when backlog age falls below this (hysteresis anti-flapping). |
+| `BACKPRESSURE__RETRY_AFTER_SECONDS` | `60` | `Retry-After` hint (seconds) while rejecting. |
+| `BACKPRESSURE__FAIL_CLOSED_ON_UNREACHABLE` | `true` | Treat unreachable probe (retries exhausted) as backlog-exceeded. |
+| `BACKPRESSURE__REJECT_ON_ANY_DEGRADED` | `false` | Legacy escape hatch: reject when any component is degraded. |
+| `BACKPRESSURE__UNHEALTHY_CHECK_INTERVAL_SECONDS` | `5.0` | Poll period while REJECTING. |

streamgate-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,40 @@
+# Contributing
+
+Thanks for your interest in contributing to streamgate!
+
+## Development setup
+
+This project uses [uv](https://docs.astral.sh/uv/) for dependency management.
+
+```bash
+git clone https://github.com/pwg-code/streamgate
+cd streamgate
+uv sync            # creates .venv and installs the project (editable) + dev tools
+```
+
+## Local checks
+
+CI runs the same gates; please run them locally before opening a PR:
+
+```bash
+uv run ruff check .        # lint (correctness + import sorting)
+uv run lint-imports        # architectural layering contract
+uv run pyright             # static type check
+uv build                   # wheel + sdist build
+```
+
+## Scope and conventions
+
+- **Mechanisms vs. policies.** The package owns mechanisms (admission orchestration, delivery, backpressure, self-healing). Policy choices (entity/slot semantics, schemas, storage carriers) belong to users. PRs should keep this boundary.
+- **Pure-core gate.** The package must never import `sqlalchemy`, `sqlmodel`, `redis`, `aiosqlite`, `aioodbc` or `httpx` — enforced by the import-linter forbidden contract in `pyproject.toml` (`[tool.importlinter]`). I/O strategy implementations belong in `examples/` as copy-paste code. If you add a protocol extension point, add or update a runnable example for it in the same PR.
+- **Examples are the storefront.** Code in `examples/` is held to the same quality bar as the package itself: CI runs lint and typecheck over it, and example READMEs must match reality. Rotting examples rot the project.
+- **Architecture gate.** The layering contract in `pyproject.toml` (`[tool.importlinter]`) is enforced: higher layers may import lower layers, never the reverse. If your change legitimately needs a new edge, update the contract in the same PR and explain why.
+- **Comments language.** Internal docstrings/comments are written in Chinese and kept that way. Docstrings on public API symbols (exported from `streamgate.__init__`) should be English-friendly.
+- **No test suite yet.** The project currently ships without tests (top roadmap item). Until then, please include a minimal reproduction script (or a runnable snippet) with bug reports and behavioral PRs so reviewers can verify against real data.
+- **Changelog.** User-visible changes must update `CHANGELOG.md`.
+
+## Submitting
+
+1. Fork / create a branch.
+2. Make the change, run the four local checks above.
+3. Open a pull request with a short description and (if applicable) the reproduction script.

streamgate-0.2.0/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.5
+Name: streamgate
+Version: 0.2.0
+Summary: A pure-core Kafka data pipeline framework: conditional admission, reliable delivery, pluggable sinks via typed hooks.
+Project-URL: Homepage, https://github.com/pwg-code/streamgate
+Project-URL: Repository, https://github.com/pwg-code/streamgate
+Project-URL: Changelog, https://github.com/pwg-code/streamgate/blob/main/CHANGELOG.md
+Author: Wanguo Pang
+License-Expression: MIT
+License-File: LICENSE
+Keywords: backpressure,dlq,ingestion,kafka,pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: System :: Distributed Computing
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: aiokafka>=0.10
+Requires-Dist: loguru>=0.7
+Requires-Dist: pydantic>=2.5
+Description-Content-Type: text/markdown
+
+# streamgate
+
+[![PyPI version](https://img.shields.io/pypi/v/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![Python versions](https://img.shields.io/pypi/pyversions/streamgate.svg)](https://pypi.org/project/streamgate/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/pwg-code/streamgate/blob/main/LICENSE)
+[![CI](https://github.com/pwg-code/streamgate/actions/workflows/ci.yml/badge.svg)](https://github.com/pwg-code/streamgate/actions/workflows/ci.yml)
+
+English | [中文](README.zh-CN.md)
+
+**streamgate** is a pure-core Kafka data pipeline framework: conditional admission at the door, reliable delivery through Kafka, and a single typed hook — `RecordWriter` — wherever you want the data to land.
+
+The wheel installs exactly three dependencies (`aiokafka`, `loguru`, `pydantic`) and contains **zero database, Redis or HTTP-client code**. All I/O strategies ship as copy-paste examples in [`examples/`](examples/).
+
+- 中文提示：本仓库对外文档以英文为主；内部代码注释保留中文。
+
+## Why streamgate
+
+- **Mechanisms in the framework, policies in your hands.** The framework owns admission *orchestration*, backpressure, health probing, reconnection, offset handling, batch buffering and graceful shutdown. You own entity/slot semantics, schemas, and — critically — **the storage carriers**: your database, your Redis, your HTTP probe are injected via typed protocols, never imported by the core.
+- **A uniqueness contract without a mandated store.** The admission flow (existence check → atomic slot reservation → cold-entity backfill → consumer-side authoritative refresh) is framework machinery. Whether the existence store is Redis, an in-process dict, or something else is your policy: implement `AdmissionPolicy`, or use the built-in zero-I/O shortcuts (`"none"`, `"in-memory"`). A production-grade Redis reference lives in [`examples/redis_admission/`](examples/redis_admission/).
+- **One sink hook, any storage.** `ConsumeSpec(sink=...)` takes a `RecordWriter` — write to SQL, Elasticsearch, another service, a file; the framework handles batching, retries, error classification and DLQ bisection around it. No database is built in, on purpose.
+- **Error classification as a hook.** Transient vs. poison vs. fatal is storage-specific knowledge. Inject an `ErrorClassifier`; the default only knows generic exceptions (and maps unknowns to poison — probe-protected bisect, never silent drops). A production-hardened SQLAlchemy classifier is in [`examples/sqlite_sink/`](examples/sqlite_sink/).
+- **Backpressure with hysteresis.** Ingest probes consumer health on a side channel and rejects based on backlog age with separate trip/recover thresholds. The core ships zero-I/O signals (`ManualBackpressureSignal`); the HTTP-probe reference is in [`examples/http_probe/`](examples/http_probe/).
+- **DLQ bisection instead of head-of-line blocking.** A poison message never wedges a partition: the failing batch is bisected with probe comparison, the bad record is quarantined to a dead-letter topic, the rest commits.
+
+## Quick Start
+
+A complete ingest → Kafka → consume → SQLite round trip that runs on a **bare install** (stdlib sink, zero extra packages) — [`examples/pure_pipeline/`](examples/pure_pipeline/):
+
+```python
+import asyncio
+from pydantic import BaseModel
+
+from streamgate import (
+    BackpressureConfig, ConsumeSpec, ConsumerConfig, ConsumerWorker,
+    IngestBinding, IngestGateway, KafkaConfig,
+)
+
+KAFKA = KafkaConfig(bootstrap_servers="localhost:29092", topic="orders")
+
+class OrderIn(BaseModel):          # ingress schema (policy lives with you)
+    order_id: str
+    amount: float
+
+binding = IngestBinding(
+    message_type="order",
+    entity_key=lambda r: r.order_id,
+    slot_key=lambda r: "order",
+    summary=lambda r: {"amount": r.amount},
+    admission="in-memory",         # built-in zero-I/O uniqueness (single process)
+)
+
+async def ingest_one(record: OrderIn) -> None:
+    gateway = IngestGateway(binding=binding, kafka_config=KAFKA,
+                            backpressure_config=BackpressureConfig(enabled=False))
+    await gateway.start()
+    outcome = await gateway.process(record)     # ACCEPTED / CONFLICT / BACKPRESSURE / ...
+    await gateway.close()
+    print(outcome.kind)
+
+async def handle(records, context) -> None:     # your storage, any library
+    ...
+
+async def consume() -> None:
+    spec = ConsumeSpec(on_record=handle, expected_message_type="order")
+    worker = ConsumerWorker(spec, kafka_config=KAFKA,
+                            consumer_config=ConsumerConfig(group_id="order-sink"))
+    await worker.run()             # blocks until SIGINT/SIGTERM
+```
+
+## Core concepts
+
+| Tier | What you use | When |
+|------|--------------|------|
+| **0 — Declarative** | `IngestBinding` (ingest side) + `ConsumeSpec` with a `sink` or `on_record` (sink side) | The common path: declare message type and uniqueness keys; inject your writer or handler |
+| **1 — Component swap** | Protocols in `streamgate.protocols` + built-in zero-I/O implementations | Replace admission, backpressure signal, codec, error classifier with your own |
+| **2 — Escape hatch** | `ConsumeSpec.on_record` / `ConsumeContext` | Handle records yourself; no sink machinery at all |
+
+Key protocols (all in `streamgate.protocols`, the frozen contract):
+
+- `AdmissionPolicy` — decide admit / 409-conflict / reject per record before it reaches Kafka (existence check + atomic reservation + cold backfill are *your* implementation against *your* store).
+- `RecordWriter` — where consumed batches land (database, search index, another service...). The only persistence path.
+- `BackpressureSignal` — tells the gateway whether to accept; built-ins are zero-I/O, dynamic probes are injected.
+- `ErrorClassifier` — RETRY / POISON / FATAL mapping for write failures.
+- `BackfillSource` — cold-entity load for admission policies that need one.
+
+**Stability:** v0.x, experimental. The protocols module is the frozen contract; everything else may still shift.
+
+## Architecture
+
+```
+your HTTP app (presentation is yours: auth/routing/OpenAPI)
+        │
+        ▼
+  IngestGateway ──► BackpressureSignal (injected: manual / HTTP probe)
+        │          AdmissionPolicy (injected: none / in-memory /
+        │           your Redis|DB-backed implementation)
+        ▼
+      Kafka ◄──────────────────────────────────────────────┐
+        │                                                  │ DLQ ◄─ poisoned records
+        ▼                                                  │      (bisect + probe compare)
+  ConsumerWorker ──► RecordWriter (your storage) ──────────┘
+        │             └─ ErrorClassifier (injected)
+        └─ health snapshot (expose it with your own web framework)
+```
+
+The package contains **zero web-framework code** (no fastapi/uvicorn) and **zero DB/Redis code** — enforced by an import-linter forbidden contract in CI.
+
+## Installation
+
+```bash
+pip install streamgate    # aiokafka + loguru + pydantic. That's it.
+```
+
+No extras. For storage/backpressure carriers, copy from `examples/` and add the
+third-party library your implementation needs (e.g. `redis`, `sqlalchemy`,
+`httpx`) to *your* project:
+
+| Example | Demonstrates | Extra deps for the example |
+|---------|--------------|----------------------------|
+| [`pure_pipeline/`](examples/pure_pipeline/) (main) | bare-install full round trip, `in-memory` admission, `on_record` stdlib SQLite sink | none |
+| [`sqlite_sink/`](examples/sqlite_sink/) | `RecordWriter` injection with the SQL upsert sugar + DB error classifier | `sqlalchemy`, `sqlmodel`, `aiosqlite` |
+| [`redis_admission/`](examples/redis_admission/) | `AdmissionPolicy` injection with Redis existence cache | `redis` |
+| [`http_probe/`](examples/http_probe/) | `BackpressureSignal` injection with hysteresis | `httpx` |
+
+Other databases (PostgreSQL, MySQL, ...): implement `RecordWriter` (Tier 1) — the same hook every built path uses.
+
+## Configuration
+
+All components configure via typed objects that map 1:1 to environment variables (`KAFKA__*`, `CONSUMER__*`, `BACKPRESSURE__*`). Required settings (e.g. the Kafka topic, consumer group id) fail fast at startup with fix instructions. Database/Redis connection settings belong to your application (see the examples' local configs). See [CONFIGURATION.md](CONFIGURATION.md).
+
+## Examples
+
+See [`examples/`](examples/) for four runnable pipelines plus a `docker-compose.yml` (Kafka + Redis). Start with [`examples/pure_pipeline/`](examples/pure_pipeline/) — it runs with nothing but `pip install streamgate`.
+
+## Roadmap
+
+- Test suite (first release ships without tests; APIs are exercised in production but the project considers this its top debt)
+- Documentation site
+- More admission/sink reference implementations in `examples/`
+
+## License
+
+[MIT](LICENSE)