mq-bridge-py-basic 0.3.8__tar.gz → 0.3.9__tar.gz

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/.gitignore

@@ -48,3 +48,7 @@ tests/**/*.pkcs12
 .pytest_cache/
 python/mq-bridge-py/.venv/
 python/mq-bridge-py/dist/
+
+# Local build setup (shared cargo target dir with mq-bridge-app), not for git
+.cargo/
+.vscode/

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/Cargo.lock

@@ -2976,6 +2976,15 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.49"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "libmqm-constants"
 version = "0.8.4"
@@ -3200,6 +3209,15 @@ dependencies = [
  "sketches-ddsketch",
 ]
 
+[[package]]
+name = "mimalloc"
+version = "0.1.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -3330,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3411,7 +3429,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3425,12 +3443,13 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "anyhow",
  "async-trait",
  "bytes",
  "fast-uuid-v7",
+ "mimalloc",
  "mq-bridge",
  "mq-bridge-bindings-common",
  "napi",
@@ -3446,12 +3465,13 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.3.8"
+version = "0.3.9"
 dependencies = [
  "anyhow",
  "async-trait",
  "bytes",
  "fast-uuid-v7",
+ "mimalloc",
  "mq-bridge",
  "mq-bridge-bindings-common",
  "pyo3",
@@ -3725,9 +3745,9 @@ dependencies = [
 
 [[package]]
 name = "omq-proto"
-version = "0.24.1"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9335ef21d570294464c491837af5477caddb05cf3c7223af1c4341a127e6035"
+checksum = "856d18f6b58f39153fcf0b574efe3f04362ff44d3a46829eb984c1f9ef2d121c"
 dependencies = [
  "bytes",
  "patricia_tree",
@@ -3739,9 +3759,9 @@ dependencies = [
 
 [[package]]
 name = "omq-tokio"
-version = "0.20.1"
+version = "0.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a9d9c6caa88d29df5b9dc9638ea9f0f33b087ecc0f149b70e61149f97bcdb80"
+checksum = "98aaa33702e7d723de268dd9fe725f93d469afef3f1bedb14cfa7f579f61a7b7"
 dependencies = [
  "arc-swap",
  "bytes",
@@ -7191,9 +7211,9 @@ dependencies = [
 
 [[package]]
 name = "yring"
-version = "0.3.10"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39146d2147cfb9f71b9f071f93f7b5dbcfb28eee528dafcaf5f500f4917cb69f"
+checksum = "f8545becda6379f37e13fbdf5d9de464849bdd1565a995368648f11fe9d96d13"
 dependencies = [
  "loom",
 ]

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.8"
+version = "0.3.9"
 
 [workspace.dependencies]
 anyhow = "1.0"
@@ -210,7 +210,7 @@ float-roundtrip = ["serde_json/float_roundtrip"]
 # runtime the client is loaded lazily only when an ibm-mq endpoint is opened
 # (clear error if the client isn't installed). Use `ibm-mq-static` instead for
 # static linking.
-full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "clickhouse", "websocket", "postgres-cdc", "object-store", "rustls-aws-lc", "ibm-mq"]
+full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "zeromq-omq", "redis-streams", "sled", "grpc", "sqlx", "clickhouse", "websocket", "postgres-cdc", "object-store", "rustls-aws-lc", "ibm-mq"]
 
 # Endpoints that build across most OS targets (glibc/musl/Alpine, macOS,
 # Windows, aarch64/armv7) without a special toolchain — no librdkafka, protoc,

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.3.8
+Version: 0.3.9
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/README.md

@@ -33,7 +33,7 @@ If you need to move data reliably between systems and you write code (Rust, Pyth
 *   **TLS everywhere, one config shape**: a single `TlsConfig` block (CA bundle, client cert/key for mTLS, insecure-skip) is reused across transports.
 *   **Self-hosted, no daemon**: generate config in the optional UI, paste it into your code, run it in-process. No hosted control plane, no separate scheduler.
 
-> **Throughput & footprint.** In our own benchmarks, the same engine — driven the zero-code way through [`mq-bridge-app`](https://github.com/marcomq/mq-bridge-app) — sustained ~267,000 rows/s copying 1,000,000 rows from Postgres to JSONL on commodity hardware at **~20 MiB peak RSS**, keeping it well in the range of dedicated data-movement tools. On a CSV→JSONL file conversion (1,000,000 mixed-type rows, ~116 MiB), it sustained **833,333 rows/s** at the same ~20 MiB, about **~43x faster** and **~22x leaner in memory** than Meltano (`tap-csv` → `target-jsonl`, ~19,500 rows/s / ~444 MiB). Full setup, methodology, and the exact parameters are in [`benches/ETL_BENCHMARKS.md`](benches/ETL_BENCHMARKS.md).
+> **Throughput & footprint.** In our own benchmarks, the same engine — driven the zero-code way through [`mq-bridge-app`](https://github.com/marcomq/mq-bridge-app) — on a CSV→JSONL file conversion (1,000,000 mixed-type rows, ~116 MiB) sustained **1,133,786 rows/s** at ~22 MiB, about **~58x faster** and **~20x leaner in memory** than Meltano (`tap-csv` → `target-jsonl`, ~19,500 rows/s / ~444 MiB). Full setup, methodology, and the exact parameters are in [`benches/ETL_BENCHMARKS.md`](benches/ETL_BENCHMARKS.md).
 
 
 ## Language Bindings
@@ -306,9 +306,14 @@ server-side (default one-hour window) — `insert_deduplication_token` lets you
 mq-bridge does not set one, so rely on `ReplacingMergeTree` for logical dedup and treat block-level
 dedup only as retry-safety.
 
-**Postgres CDC — deterministic id + `postgres.key`.** The `postgres_cdc` source already resumes from a
-durable LSN checkpoint (no re-delivery on clean restart). For the crash window (row written, then a
-crash before the checkpoint flush) it is at-least-once, so make the sink idempotent. Each change event
+**Postgres CDC — deterministic id + `postgres.key`.** The `postgres_cdc` source resumes from the slot's
+durable `confirmed_flush_lsn`. In-band standby feedback is asynchronous and is *not* flushed when the
+stream stops, so the last acks are made durable by the consumer's `Drop`, which stops the stream and
+advances the slot synchronously. Re-delivery is therefore avoided only on a restart that actually runs
+that teardown — a host process that exits without dropping the route (or one on a current-thread Tokio
+runtime, where the blocking advance is skipped) replays everything since the last asynchronous feedback
+tick. Set `checkpoint_store` to a `file://` path for a second, per-ack durable position that survives
+teardown regardless. Treat the source as at-least-once and make the sink idempotent. Each change event
 carries the full row (so the primary key is in the payload), `postgres.lsn` (a monotonic version),
 `postgres.operation`/`schema`/`table`, and — when the table has a primary key / replica identity —
 `postgres.key` (the key value). The event's `message_id` is a stable hash of `schema.table + key +

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/REFERENCE.md

@@ -17,7 +17,7 @@ endpoints, shape routing, or terminate a request. Data endpoints (`kafka`, `nats
 Middleware attaches to an endpoint via a `middlewares:` list, on the **input**, the
 **output**, or both:
 
-```yaml
+```yaml route
 my_route:
   input:
     middlewares:
@@ -38,6 +38,11 @@ layer and sees the failures of the ones before it.** Put `dlq` last.
 **Input (consumer) middlewares are applied in reverse, so the *first* entry is outermost**
 and runs first on an incoming message.
 
+**Consequence: a route that reads back what another route wrote needs the *reversed* list.**
+Writing with `[compression, encryption]` produces `compress(encrypt(payload))`; a reader
+given that same list would try to decrypt first and fail. The reading route must say
+`[encryption, compression]`. The lists mirror — they are not copied.
+
 > This is asserted by `route::tests::test_dlq_and_retry_batch_integration`,
 > `middleware::transform::tests::test_rejected_message_reaches_the_dlq_through_the_config_wiring`,
 > and `reference_docs_test::publisher_middleware_wraps_last_entry_outermost`, and is
@@ -58,7 +63,7 @@ middlewares:
 | [`retry`](#retry) | – | ✅ | – | Exponential-backoff retry of failed sends |
 | [`dlq`](#dlq) | – | ✅ | – | Route permanently-failed messages to another endpoint |
 | [`transform`](#transform) | ✅ | ✅ | – | Declarative JSON mapping, coercion, validation |
-| [`deduplication`](#deduplication) | ✅ | – | `dedup` | Drop repeated message IDs within a TTL |
+| [`deduplication`](#deduplication) | ✅ | – | `dedup` | Drop repeated keys within a TTL |
 | [`weak_join`](#weak_join) | ✅ | – | – | Correlate and join related messages |
 | [`buffer`](#buffer) | ✅ | ✅ | – | Coalesce single sends into batches |
 | [`limiter`](#limiter) | ✅ | ✅ | – | Cap throughput to a message rate |
@@ -103,7 +108,7 @@ Retries failed sends with exponential backoff. Output only.
 | `max_interval_ms` | integer | `5000` |
 | `multiplier` | float | `2.0` |
 
-```yaml
+```yaml middleware
 - retry: { max_attempts: 5, initial_interval_ms: 200, max_interval_ms: 10000, multiplier: 2.0 }
 ```
 
@@ -119,7 +124,7 @@ Sends permanently-failed messages to a separate endpoint instead of failing the
 |---|---|---|
 | `endpoint` | Endpoint | yes |
 
-```yaml
+```yaml middleware
 - dlq:
     endpoint:
       file: { path: "dead-letters.jsonl" }
@@ -161,7 +166,7 @@ validation — over a single parse. Input and output.
 `schema` and `schema_file` are mutually exclusive. A mapping rule is either a bare path
 string or `{ path, default, required }`.
 
-```yaml
+```yaml middleware
 - transform:
     mapping:
       firstName: "$.first_name"
@@ -185,7 +190,7 @@ lossless ones: `string → integer`, `string → number`, `string → boolean` (
 A field carrying a JSON document as a string is decoded by `contentMediaType`, following
 JSON Schema 2020-12:
 
-```yaml
+```yaml middleware
 - transform:
     schema:
       type: object
@@ -227,7 +232,7 @@ with neither stage configured leaves the payload untouched without parsing it.
 
 ### `deduplication`
 
-Drops messages whose ID was already seen within the TTL. Input only. Requires the `dedup`
+Drops messages whose key was already seen within the TTL. Input only. Requires the `dedup`
 feature (pulls `sled`).
 
 | Field | Type | Required |
@@ -235,13 +240,22 @@ feature (pulls `sled`).
 | `store` | string | one of `store`/`sled_path` |
 | `sled_path` | string | one of `store`/`sled_path` |
 | `ttl_seconds` | integer | yes |
+| `key` | string | no (defaults to `message_id`) |
+
+`key` is an interpolation template (see `${namespace:selector}`), typically
+`"${payload:order_id}"`. Without it the key is the `message_id`, which most sources
+regenerate on every read — so re-reading the same source deduplicates nothing and only
+in-flight redeliveries are suppressed. Set `key` to a business key whenever you need
+dedup to survive a re-read.
 
 `store` selects the backend by URL scheme:
 
 - `sled:///path` (or a bare path) — a local sled database; per-process, not cluster-wide.
 - `mongodb://host/db[/collection]` — a shared collection, so multiple instances of a route
-  deduplicate against one another. Requires the `mongodb` feature. Entries expire via a
-  MongoDB TTL index; the collection defaults to `mqb_dedup_<route>`. Point it at the same
+  deduplicate against one another. Requires the `mongodb` feature. Expiry is judged on read,
+  so a `ttl_seconds` boundary is honoured exactly; the TTL index only reclaims space
+  afterwards (MongoDB's sweep can lag by up to a minute). The collection defaults to
+  `mqb_dedup_<route>`. Point it at the same
   deployment your sink already uses to avoid running extra infrastructure.
 - `postgres|mysql|mariadb|sqlite://…[/table]` — a shared SQL table (`dedup_key` PK,
   `expire_at`), so multiple instances deduplicate against one another. Requires the `sqlx`
@@ -250,18 +264,22 @@ feature (pulls `sled`).
 
 `sled_path` is the legacy spelling of a local sled store and is equivalent to `store: "sled://<path>"`.
 
-```yaml
+```yaml middleware
 - deduplication: { store: "sled:///var/lib/mq-bridge/dedup", ttl_seconds: 3600 }
 ```
 
-```yaml
+```yaml middleware
 - deduplication: { store: "mongodb://localhost:27017/etl", ttl_seconds: 3600 }
 ```
 
-```yaml
+```yaml middleware
 - deduplication: { store: "postgres://user:pass@localhost/etl", ttl_seconds: 3600 }
 ```
 
+```yaml middleware
+- deduplication: { store: "sled:///var/lib/mq-bridge/dedup", ttl_seconds: 3600, key: "${payload:order_id}" }
+```
+
 When MongoDB is your sink and messages carry a business key, prefer the sink's own unique
 index (`id_field` on the mongodb output) over this middleware — the target collection then
 *is* the deduplication authority, with no second write. See the idempotency notes in README.
@@ -279,7 +297,7 @@ Correlates messages by a metadata key and emits them as one joined message. Inpu
 | `required` | list of branch names | `[]` |
 | `on_timeout` | `fire` \| `discard` | `fire` |
 
-```yaml
+```yaml middleware
 # Count mode: wait for any 3 messages sharing a correlation_id, emit a JSON array.
 - weak_join: { group_by: "correlation_id", expected_count: 3, timeout_ms: 5000 }
 
@@ -307,7 +325,7 @@ Accumulates single sends and forwards them as one batch. Input and output.
 | `max_messages` | integer | yes |
 | `max_delay_ms` | integer | yes |
 
-```yaml
+```yaml middleware
 - buffer: { max_messages: 500, max_delay_ms: 20 }
 ```
 
@@ -322,7 +340,7 @@ Paces throughput to a target rate. Input and output.
 |---|---|---|
 | `messages_per_second` | float (> 0) | yes |
 
-```yaml
+```yaml middleware
 - limiter: { messages_per_second: 250 }
 ```
 
@@ -336,7 +354,7 @@ Sleeps a fixed duration before each receive or send. Input and output.
 |---|---|---|
 | `delay_ms` | integer | yes |
 
-```yaml
+```yaml middleware
 - delay: { delay_ms: 100 }
 ```
 
@@ -356,7 +374,7 @@ Persists HTTP cookies and arbitrary session values across messages. Input and ou
 | `export_metadata_prefix` | string | – |
 | `inject_metadata` | map string→string | `{}` |
 
-```yaml
+```yaml middleware
 - cookie_jar:
     shared_scope: "login-session"
     capture_metadata_keys: ["x-csrf-token"]
@@ -380,7 +398,7 @@ output. Requires the `encryption` feature.
 | `key` | string — base64-encoded 32-byte key; `${env:VAR}` reads it from the environment | required |
 | `decrypt_keys` | map key_id → key | `{}` |
 
-```yaml
+```yaml middleware
 - encryption: { key: "${env:MQB_ENC_KEY}" }
 ```
 
@@ -402,7 +420,7 @@ Do **not** combine this middleware with a sink's batch `compression` on the same
 ciphertext does not compress. For compressed *and* encrypted data at rest, use the `file` /
 `object_store` endpoints' own fields instead, which apply compress-then-encrypt per batch:
 
-```yaml
+```yaml endpoint
 output:
   file:
     path: "data.enc"
@@ -443,7 +461,7 @@ feature.
 | `algorithm` | `none` \| `gzip` \| `lz4` \| `zstd` | `zstd` |
 | `max_decompressed_bytes` | integer — reject a payload that decompresses larger than this (bomb guard); consumer side only | unset (no limit) |
 
-```yaml
+```yaml middleware
 - compression: { algorithm: zstd }
 ```
 
@@ -464,7 +482,7 @@ endpoints' own `compression`/`encryption` fields instead.
 Emits throughput, latency and error metrics for the endpoint. Input and output. Requires the
 `metrics` feature. Takes no options; its presence enables collection.
 
-```yaml
+```yaml middleware
 - metrics: {}
 ```
 
@@ -480,7 +498,7 @@ Deliberate fault injection for testing recovery paths. Input and output.
 | `trigger_on_message` | integer (1-indexed) | – (every message) |
 | `enabled` | bool | `true` |
 
-```yaml
+```yaml middleware
 - random_panic: { mode: disconnect, trigger_on_message: 500 }
 ```
 
@@ -488,6 +506,24 @@ Deliberate fault injection for testing recovery paths. Input and output.
 non-retryable one — useful for exercising a `dlq`. Keep `enabled: false` in committed configs
 rather than deleting the block.
 
+The middleware block alone is **not** enough: fault injection is gated per route by
+`allow_fault_injection`, which defaults to `false`. Copying only the snippet above leaves the
+middleware inert (the route logs that it is disabled). A complete, working configuration:
+
+```yaml route
+flaky_test_route:
+  allow_fault_injection: true
+  input:
+    memory: { topic: "in" }
+    middlewares:
+      - random_panic: { mode: disconnect, trigger_on_message: 500 }
+  output:
+    memory: { topic: "out" }
+```
+
+`allow_fault_injection: true` is intended for test configurations only. Do not enable it — or
+the `random_panic` middleware — in production configs.
+
 ### `custom` (middleware)
 
 Delegates to a factory you registered programmatically.
@@ -497,7 +533,7 @@ Delegates to a factory you registered programmatically.
 | `name` | string | yes |
 | `config` | any JSON | yes |
 
-```yaml
+```yaml middleware
 - custom:
     name: "my_enricher"
     config: { lookup_url: "http://enrich.internal" }
@@ -545,7 +581,7 @@ use mq_bridge::route::register_endpoint;
 register_endpoint("common_queue", Endpoint::new_memory("shared_memory_topic", 100));
 ```
 
-```yaml
+```yaml route
 enrich:
   input: { ref: "common_queue" }
   output: { nats: { subject: "enriched", url: "nats://localhost:4222" } }
@@ -564,7 +600,7 @@ nesting depth is bounded.
 
 Publishes each message to every listed endpoint. Output only.
 
-```yaml
+```yaml endpoint
 output:
   fanout:
     - kafka: { topic: "audit", url: "localhost:9092" }
@@ -585,7 +621,7 @@ Content-based routing: picks a destination by the value of a **metadata key**.
 | `cases` | map value → Endpoint | yes |
 | `default` | Endpoint | no |
 
-```yaml
+```yaml endpoint
 output:
   switch:
     metadata_key: "http_status_code"
@@ -611,7 +647,7 @@ turning a request/reply exchange into a one-way flow.
 | `to` | Endpoint (request-capable) | yes |
 | `forward_to` | Endpoint | yes |
 
-```yaml
+```yaml endpoint
 output:
   request:
     to: { http: { url: "https://api.internal/score" } }
@@ -628,7 +664,7 @@ status key such as `http_status_code`.
 Replies to the origin of the current request. Output only, and the recommended way to build
 request/reply routes.
 
-```yaml
+```yaml route
 http_echo:
   input: { http: { url: "0.0.0.0:8080" } }
   output: { response: {} }
@@ -644,7 +680,7 @@ pipeline. See [README.md](README.md#patterns-request-response).
 An output endpoint that **ignores the incoming payload** and instead reads one message from
 the wrapped consumer, returning it as the response. The inbound message is purely a trigger.
 
-```yaml
+```yaml route
 # HTTP GET pulls the next message off a Kafka topic.
 poll_api:
   input: { http: { url: "0.0.0.0:8080", method: "GET" } }
@@ -670,7 +706,7 @@ source).
 
 Accepts either a bare string or the full map form:
 
-```yaml
+```yaml endpoint
 output: { static: "OK" }                       # shorthand, body JSON-encoded
 
 output:
@@ -707,7 +743,7 @@ structure — append `| raw` to a token to splice it verbatim. To emit a literal
 `${…}`, write `$${…}` (a bare `$$` is left as-is); any `${…}` with an unknown namespace is also
 left untouched.
 
-```yaml
+```yaml endpoint
 output:
   static:
     body: '{"error":"not found","id":"${message:id}","at":"${gen:now}"}'
@@ -726,7 +762,7 @@ bodies between routes.
 | `correlation_id` | string | **required on consumers, must be unset on publishers** |
 | `capacity` | integer | default `100`, per partition |
 
-```yaml
+```yaml endpoint
 output:
   stream_buffer: { topic: "responses" }        # publisher: no correlation_id
 
@@ -741,7 +777,7 @@ and ignores it. Primarily wired up via `HttpConfig::stream_response_to`.
 
 Discards every message. Output only. This is the **default output** when a route omits one.
 
-```yaml
+```yaml route
 drain:
   input: { kafka: { topic: "noisy", url: "localhost:9092" } }
   output: null          # a bare YAML null
@@ -762,7 +798,7 @@ Delegates to a factory you registered programmatically.
 | `name` | string | yes |
 | `config` | any JSON | yes |
 
-```yaml
+```yaml endpoint
 output:
   custom:
     name: "my_sink"

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/benches/ETL_BENCHMARKS.md

@@ -72,7 +72,7 @@ theirs. The run brief lives in [`mq-bridge-app-benchmark-prompt.md`](mq-bridge-a
 
 | Scenario | Payload | Batch | Concurrency | Source → Sink | Result |
 | --- | --- | --- | --- | --- | --- |
-| Bulk-copy throughput | 7-col mixed-type rows, 1,000,000 rows | 1024 | 4 | postgres (keyset cursor) → file (JSONL) | **303,398 rows/s** (3.296 s) |
+| Bulk-copy throughput | 7-col mixed-type rows, 1,000,000 rows | 1024 | 4 | postgres (keyset cursor) → file (JSONL) | **384,615 rows/s** (2.600 s) |
 
 Command: `mq-bridge-app copy --from postgres://…?table=…&cursor_column=id --to file://…?format=raw --drain --batch-size 1024 --concurrency 4`, wall-clocked. Requires an index on the cursor column (`CREATE INDEX ON <table>(id)`) — without it the keyset-pagination reader (`WHERE id > $cursor ORDER BY id LIMIT batch`) does a full scan per batch (near-quadratic).
 
@@ -82,10 +82,10 @@ Same source table (`bench`: 1,000,000 rows, 7 mixed-type columns), same Postgres
 
 | Scenario | Payload | Batch | Concurrency | Source → Sink | Throughput | Peak RSS |
 | --- | --- | --- | --- | --- | --- | --- |
-| mq-bridge-app `copy` | 7-col, 1,000,000 rows | 1024 | 1 | postgres (keyset cursor) → file (JSONL) | **266,951 rows/s** | 19.9 MiB |
+| mq-bridge-app `copy` | 7-col, 1,000,000 rows | 1024 | 1 | postgres (keyset cursor) → file (JSONL) | **338,066 rows/s** | 39.8 MiB |
 | Meltano (`tap-postgres` → `target-jsonl`) | 7-col, 1,000,000 rows | default Singer config | — | postgres → JSONL | 15,356 rows/s | 599.7 MiB |
 
-**~17.4x faster and ~30x leaner in peak memory** than Meltano in this scenario. Full setup (including the Meltano project config) is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#5--postgres--jsonl-vs-meltano-tap-postgres--target-jsonl).
+**~22x faster and ~15x leaner in peak memory** than Meltano in this scenario. Full setup (including the Meltano project config) is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#5--postgres--jsonl-vs-meltano-tap-postgres--target-jsonl).
 
 ### Preliminary run — CSV → JSONL vs. Meltano
 
@@ -93,10 +93,10 @@ Same seeded dataset both sides, same machine, one-shot full-file CSV → local J
 
 | Scenario | Payload | Batch | Concurrency | Source → Sink | Throughput | Peak RSS |
 | --- | --- | --- | --- | --- | --- | --- |
-| mq-bridge-app `copy` | 7-col mixed-type, 1,000,000 rows (~116 MiB) | 1024 | 1 | file (CSV) → file (JSONL) | **833,333 rows/s** | 20.0 MiB |
+| mq-bridge-app `copy` | 7-col mixed-type, 1,000,000 rows (~116 MiB) | 1024 | 1 | file (CSV) → file (JSONL) | **1,133,786 rows/s** | 21.9 MiB |
 | Meltano (`tap-csv` → `target-jsonl`) | 7-col mixed-type, 1,000,000 rows | default Singer config | — | file (CSV) → JSONL | ~19,500 rows/s | 443.8 MiB |
 
-**~43x faster and ~22x leaner in peak memory** than Meltano in this scenario. Full setup is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#6--csv--jsonl-vs-meltano).
+**~58x faster and ~20x leaner in peak memory** than Meltano in this scenario. Full setup is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#6--csv--jsonl-vs-meltano).
 
 ## Status
 

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/mq-bridge.schema.json

@@ -478,6 +478,14 @@
       "description": "Deduplication middleware configuration.\n\nPrevents duplicate messages from being processed using a sled, MongoDB, or SQL backend.\nMessages are identified by their deduplication key and removed after the TTL expires.",
       "type": "object",
       "properties": {
+        "key": {
+          "description": "Dedup key template, e.g. `${payload:order_id}`. Defaults to `message_id`.",
+          "type": [
+            "string",
+            "null"
+          ],
+          "default": null
+        },
         "sled_path": {
           "description": "Local Sled directory (legacy). Prefer `store`.",
           "type": [
@@ -922,9 +930,16 @@
           ]
         },
         {
-          "type": "string",
+          "type": "object",
           "format": "structural_endpoint",
-          "const": "null"
+          "properties": {
+            "null": {
+              "type": "null"
+            }
+          },
+          "required": [
+            "null"
+          ]
         }
       ],
       "unevaluatedProperties": false
@@ -963,7 +978,7 @@
       "type": "object",
       "properties": {
         "compression": {
-          "description": "Per-batch compression (`none`, `gzip`, `lz4`, `zstd`). Requires the `compression` feature; consume mode only.",
+          "description": "Per-batch compression (`none`, `gzip`, `lz4`, `zstd`). Requires the `compression` feature. Publishers: always. Consumers: must match, and only the default `consume` mode reads it.",
           "$ref": "#/$defs/Compression",
           "default": "none"
         },
@@ -975,7 +990,7 @@
           ]
         },
         "encryption": {
-          "description": "At-rest AEAD encryption applied after compression. Requires the `encryption` feature; consume mode only.",
+          "description": "At-rest AEAD encryption applied after compression. Requires the `encryption` feature. Publishers: always. Consumers: must match, and only the default `consume` mode reads it.",
           "anyOf": [
             {
               "$ref": "#/$defs/EncryptionConfig"
@@ -2609,7 +2624,7 @@
           "minimum": 0
         },
         "temporary_slot": {
-          "description": "Use a temporary slot (dropped on disconnect). Not restart-safe; default is a permanent slot.",
+          "description": "Ephemeral run: drop the slot when the route stops. Not restart-safe; a hard crash leaks it.",
           "type": "boolean",
           "default": false
         },

{mq_bridge_py_basic-0.3.8 → mq_bridge_py_basic-0.3.9}/mq_bridge/config.pyi

@@ -86,6 +86,7 @@ class DeadLetterQueueMiddleware(TypedDict, total=False):
 
 class DeduplicationMiddleware(TypedDict, total=False):
     """Deduplication middleware configuration."""
+    key: Optional[str]
     sled_path: Optional[str]
     store: Optional[str]
     ttl_seconds: Required[int]
@@ -113,7 +114,36 @@ class EncryptionConfig(TypedDict, total=False):
 
 class Endpoint(TypedDict, total=False):
     """Represents a connection point for messages, which can be a source (input) or a sink (output)."""
+    amqp: AmqpConfig
+    aws: AwsConfig
+    clickhouse: ClickHouseConfig
+    custom: Dict[str, Any]
+    fanout: List[Endpoint]
+    file: FileConfig
+    grpc: GrpcConfig
+    http: HttpConfig
+    ibmmq: IbmMqConfig
+    kafka: KafkaConfig
+    memory: MemoryConfig
     middlewares: List[Middleware]
+    mongodb: MongoDbConfig
+    mqtt: MqttConfig
+    nats: NatsConfig
+    null: Any
+    object_store: ObjectStoreConfig
+    postgres_cdc: PostgresCdcConfig
+    reader: Endpoint
+    redis_streams: RedisStreamsConfig
+    ref: str
+    request: RequestForwardConfig
+    response: ResponseConfig
+    sled: SledConfig
+    sqlx: SqlxConfig
+    static: StaticConfig
+    stream_buffer: StreamBufferConfig
+    switch: SwitchConfig
+    websocket: WebSocketConfig
+    zeromq: ZeroMqConfig
 
 
 class FileConfig(TypedDict, total=False):