mq-bridge-py-basic 0.3.4__tar.gz → 0.3.5__tar.gz

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/CONFIGURATION.md

@@ -116,6 +116,8 @@ zeromq_pipeline:
       url: "tcp://localhost:5556"
       socket_type: "push"
       bind: false
+      # format: "raw"         # send/receive raw payload bytes per frame instead of JSON-wrapping (e.g. JPEG, Protobuf)
+      # format: "raw_framed"  # like "raw" but prepends a JSON metadata frame so headers still travel. Default "json".
 
 # Route 8: PostgreSQL via SQLx
 sqlx_postgres_route:

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/Cargo.lock

@@ -1002,6 +1002,7 @@ dependencies = [
  "iana-time-zone",
  "js-sys",
  "num-traits",
+ "serde",
  "wasm-bindgen",
  "windows-link",
 ]
@@ -1279,6 +1280,16 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
 
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin 0.10.1",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -1771,9 +1782,9 @@ dependencies = [
 
 [[package]]
 name = "fast-uuid-v7"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07a7bb9c4e64a101eeec4ff43c3974178e3dd2bff832fc2c2d6edeaa14c449e5"
+checksum = "e8eaa2987e8b03da0c1659e70392d20480a595047e3db7922094ffe364e9d7b0"
 dependencies = [
  "rand 0.9.4",
 ]
@@ -1826,7 +1837,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
 dependencies = [
  "futures-core",
  "futures-sink",
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -1837,7 +1848,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be"
 dependencies = [
  "futures-core",
  "futures-sink",
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -2344,6 +2355,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+
 [[package]]
 name = "hybrid-array"
 version = "0.4.12"
@@ -2679,6 +2696,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -3154,7 +3180,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -3187,6 +3213,7 @@ dependencies = [
  "mongodb",
  "mq-bridge",
  "mqi",
+ "object_store",
  "once_cell",
  "percent-encoding",
  "pgwire-replication",
@@ -3227,7 +3254,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3241,7 +3268,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3262,7 +3289,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.3.4"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3490,6 +3517,46 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "object_store"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
+dependencies = [
+ "async-trait",
+ "aws-lc-rs",
+ "base64",
+ "bytes",
+ "chrono",
+ "crc-fast",
+ "form_urlencoded",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http 1.4.2",
+ "http-body-util",
+ "httparse",
+ "humantime",
+ "hyper 1.10.1",
+ "itertools 0.15.0",
+ "md-5 0.11.0",
+ "parking_lot 0.12.5",
+ "percent-encoding",
+ "quick-xml",
+ "rand 0.10.1",
+ "reqwest",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "url",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
 [[package]]
 name = "oid-registry"
 version = "0.8.1"
@@ -4157,6 +4224,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -5328,6 +5405,12 @@ dependencies = [
  "lock_api",
 ]
 
+[[package]]
+name = "spin"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
+
 [[package]]
 name = "spki"
 version = "0.7.3"

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/Cargo.toml

@@ -16,13 +16,13 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.4"
+version = "0.3.5"
 
 [workspace.dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
 bytes = "1"
-fast-uuid-v7 = "0.1.5"
+fast-uuid-v7 = "0.1.6"
 serde = "1.0"
 serde_json = "1.0"
 serde_yaml_ng = "0.10"
@@ -97,6 +97,10 @@ lapin = { version = "4.10", default-features = false, features = ["tokio", "rust
 # MongoDB
 mongodb = { version = "3.6", optional = true, features = ["cert-key-password"] }
 
+# Object store (checkpoint backend): cloud REST via reqwest, not aws-sdk. 0.12 is the
+# last release supporting rust-version 1.75 (0.14 needs 1.85).
+object_store = { version = "0.14", optional = true, default-features = false, features = ["aws", "gcp", "azure", "http"] }
+
 # NATS
 async-nats = { version = "0.49", optional = true }
 
@@ -180,7 +184,7 @@ middleware = ["metrics", "dedup"]
 # runtime the client is loaded lazily only when an ibm-mq endpoint is opened
 # (clear error if the client isn't installed). Use `ibm-mq-static` instead for
 # static linking.
-full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "clickhouse", "websocket", "postgres-cdc", "rustls-aws-lc", "ibm-mq"]
+full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "clickhouse", "websocket", "postgres-cdc", "object-store", "rustls-aws-lc", "ibm-mq"]
 
 # Functionality features
 dedup = ["sled"]
@@ -229,6 +233,8 @@ redis-streams = ["dep:redis"]
 grpc = ["tonic", "prost", "tokio-stream", "tonic-prost", "tonic-prost-build", "protoc-bin-vendored", "rustls"]
 sqlx = ["dep:sqlx", "url"]
 clickhouse = ["dep:reqwest", "dep:flate2", "url"]
+# Cloud object-store checkpoint backend (S3/GCS/Azure/R2). Talks cloud REST via reqwest.
+object-store = ["dep:object_store", "url"]
 # Postgres logical-replication CDC (pgoutput). Reuses the sqlx dependency for the
 # slot-lifecycle control plane; the streaming wire protocol is pgwire-replication.
 # Needs Rust 1.88 (pgwire-replication MSRV), like the amqp feature gate.

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.3.4
+Version: 0.3.5
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/README.md

@@ -25,7 +25,7 @@ If you need to move data reliably between systems and you write code (Rust, Pyth
 
 **Prefer not to write code?** [`mq-bridge-app`](https://github.com/marcomq/mq-bridge-app) runs the exact same engine as a **standalone, zero-code ETL service** configured entirely by **YAML or environment variables** — move data from A to B without writing a line. It ships a **Postman-style UI** to build, send, and inspect messages against a route, and can **import Postman collections and AsyncAPI documents** to scaffold routes and endpoints for you.
 
-*   **16+ transports, one API**: Kafka, NATS, AMQP (RabbitMQ), MQTT, MongoDB, **Postgres CDC** (logical replication), PostgreSQL / MySQL / SQLite (SQLx), ClickHouse, HTTP, WebSocket, gRPC, ZeroMQ, Redis Streams, AWS SQS/SNS, IBM MQ, files, and in-memory channels — all behind the same `receive_batch` / `send_batch` shape.
+*   **16+ transports, one API**: Kafka, NATS, AMQP (RabbitMQ), MQTT, MongoDB, **Postgres CDC** (logical replication), PostgreSQL / MySQL / SQLite (SQLx), ClickHouse, HTTP, WebSocket, gRPC, ZeroMQ, Redis Streams, AWS SQS/SNS, cloud object storage (S3 / GCS / Azure), IBM MQ, files, and in-memory channels — all behind the same `receive_batch` / `send_batch` shape.
 *   **Change Data Capture**: stream row-level changes from **Postgres** (logical replication / `pgoutput`) and **MongoDB** (change streams) as flat rows with an operation marker.
 *   **Restart-safe delivery**: batch-aware ack/nack with commit sequencing for cumulative-ack brokers; the integration suite shows **no data loss during in-flight broker restarts**, including a Postgres CDC restart-safety test.
 *   **Reliability middleware, not a framework**: retries, dead-letter queues, deduplication, rate limiting, and cookie/session persistence wrap any endpoint.
@@ -33,7 +33,7 @@ If you need to move data reliably between systems and you write code (Rust, Pyth
 *   **TLS everywhere, one config shape**: a single `TlsConfig` block (CA bundle, client cert/key for mTLS, insecure-skip) is reused across transports.
 *   **Self-hosted, no daemon**: generate config in the optional UI, paste it into your code, run it in-process. No hosted control plane, no separate scheduler.
 
-> **Throughput.** In our own benchmarks, the same engine — driven the zero-code way through [`mq-bridge-app`](https://github.com/marcomq/mq-bridge-app) — sustained ~303,000 rows/s copying 1,000,000 rows from Postgres to JSONL on commodity hardware, keeping it well in the range of dedicated data-movement tools. On a CSV→JSONL file conversion (1,000,000 mixed-type rows, ~116 MiB), it sustained **833,333 rows/s**, about **~43x faster than Meltano** (`tap-csv` → `target-jsonl`, ~19,500 rows/s). Full setup, methodology, and the exact parameters are in [`benches/ETL_BENCHMARKS.md`](benches/ETL_BENCHMARKS.md).
+> **Throughput & footprint.** In our own benchmarks, the same engine — driven the zero-code way through [`mq-bridge-app`](https://github.com/marcomq/mq-bridge-app) — sustained ~267,000 rows/s copying 1,000,000 rows from Postgres to JSONL on commodity hardware at **~20 MiB peak RSS**, keeping it well in the range of dedicated data-movement tools. On a CSV→JSONL file conversion (1,000,000 mixed-type rows, ~116 MiB), it sustained **833,333 rows/s** at the same ~20 MiB, about **~43x faster** and **~22x leaner in memory** than Meltano (`tap-csv` → `target-jsonl`, ~19,500 rows/s / ~444 MiB). Full setup, methodology, and the exact parameters are in [`benches/ETL_BENCHMARKS.md`](benches/ETL_BENCHMARKS.md).
 
 
 ## Language Bindings
@@ -72,7 +72,7 @@ For implementation details and quick start examples for each usage type, see the
 
 ## Features
 
-*   **Supported Backends**: Kafka, NATS, AMQP (RabbitMQ), MQTT, MongoDB, SQL Databases (PostgreSQL, MySQL, SQLite via sqlx), **Postgres CDC** (logical replication), ClickHouse, gRPC, HTTP, WebSocket, ZeroMQ, Redis Streams, Files, AWS (SQS/SNS), IBM MQ, and in-memory channels.
+*   **Supported Backends**: Kafka, NATS, AMQP (RabbitMQ), MQTT, MongoDB, SQL Databases (PostgreSQL, MySQL, SQLite via sqlx), **Postgres CDC** (logical replication), ClickHouse, gRPC, HTTP, WebSocket, ZeroMQ, Redis Streams, Files, cloud object storage (S3 / GCS / Azure), AWS (SQS/SNS), IBM MQ, and in-memory channels.
 *   **Change Data Capture**: PostgreSQL logical replication (`postgres_cdc`) and MongoDB change streams, surfaced as flat rows with an `*.operation` marker (`insert`/`update`/`delete`/`truncate`).
     > **Note**: IBM MQ is included in the `full` feature set via the `ibm-mq` feature, which loads the IBM MQ client library at runtime via dlopen — **no IBM SDK is needed to build**. The IBM MQ redistributable client only has to be present at runtime, and only if you actually use an IBM MQ endpoint (it is loaded lazily on first connect). If the client is missing, the affected route fails fast with a non-retryable error instead of reconnecting forever. The loader finds the client via the platform default name, `MQ_INSTALLATION_PATH` (e.g. `/opt/mqm`), or an explicit `MQB_IBM_MQ_LIB` path. To link the client statically at build time instead, use the `ibm-mq-static` feature (requires the IBM MQ SDK). See the [mqi crate](https://crates.io/crates/mqi/) for details.
     >
@@ -197,6 +197,33 @@ Databases have no native pub/sub, so `mq-bridge` reads them as a source in one o
 *   **Cursor polling** — pages an existing table by a monotonic `cursor_column` (`WHERE col > $last ORDER BY col ASC`), persisting the last read value under `cursor_id`. Captures **appends only** — updates and deletes are not observed. Available on **SQLx** (PostgreSQL / MySQL / MariaDB / SQLite) and **ClickHouse**; while idle the poll interval backs off exponentially between `polling_interval_ms` and `max_polling_interval_ms`. **SQLite** and **ClickHouse** are polling-only — they have no server-side change log.
 
 
+### Cloud Object Storage (S3 / GCS / Azure)
+The `object_store` endpoint (alias `s3`) reads and writes cloud object stores — Amazon S3, Google Cloud Storage, Azure Blob, Cloudflare R2, and anything else the [`object_store`](https://crates.io/crates/object_store) crate speaks — behind the same `receive_batch` / `send_batch` API. Enable it with the `object-store` feature. Credentials and backend options are read from the process environment (`AWS_ACCESS_KEY_ID`, `AWS_ENDPOINT`, `AWS_REGION`, `GOOGLE_SERVICE_ACCOUNT`, `AZURE_STORAGE_ACCOUNT`, ...); the URL scheme picks the backend (`s3://`, `gs://`, `az://`).
+
+*   **As a sink**, each flushed batch is encoded with the same file endpoint formats (`normal` JSONL, `json`, `text`, `raw`) and written as **one immutable object** at `<prefix>/[YYYY/MM/DD/]<uuidv7>.<ext>`. Objects are write-once — nothing is appended or mutated. The uuidv7 name already sorts by write time; the optional `date_partition` prefix (on by default, derived from that same id's timestamp) is a readability / lifecycle-rule convenience.
+*   **As a source**, objects under the prefix are listed in key order, fetched whole, split on the delimiter, and emitted. Progress is a durable cursor holding the last fully-acked object key: set `cursor_id` and an external `checkpoint_store` (`file://`, `s3://`, `postgres://`, `mongodb://`) so a restart resumes without re-emitting. Objects are **never deleted or rewritten** — resume is non-destructive and at-least-once at object granularity (a nacked batch is redelivered; the cursor only advances once an object is fully acked). `csv` is supported on the source only.
+
+```yaml
+archive_to_s3:
+  input:
+    memory: { topic: "events" }
+  output:
+    object_store:
+      url: "s3://my-bucket/events"
+      format: normal            # JSONL; also json / text / raw
+
+replay_from_s3:
+  input:
+    object_store:
+      url: "s3://my-bucket/events"
+      cursor_id: "replayer-1"
+      checkpoint_store: "file:///var/lib/mqb/s3-cursor.json"
+  output:
+    nats: { subject: "events.replay", url: "nats://localhost:4222" }
+```
+
+> Point `checkpoint_store` at a **different** bucket or prefix than the source reads; a cursor object written under the source prefix would be listed and re-read as data (the source rejects an overlapping object-store checkpoint location).
+
 ### Response Endpoint
 The `response` output endpoint sends a reply back to the original requester. This is useful for synchronous request-reply flows, for example HTTP-to-NATS-to-HTTP. Use `response: {}` as the output endpoint configuration.
 

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/benches/ETL_BENCHMARKS.md

@@ -61,14 +61,14 @@ The scenarios are run through
 path a real user takes) so the numbers are comparable to how other ETL/CDC tools publish
 theirs. The run brief lives in [`mq-bridge-app-benchmark-prompt.md`](mq-bridge-app-benchmark-prompt.md).
 
-> ⚠️ **Preliminary — measured on battery power.** The number below was measured on a
-> laptop running on battery, where macOS throttles the CPU. It is **not yet authoritative**
-> and is pending a re-run on AC (mains) power before publication. Treat it as a floor, not a
-> confirmed figure.
+> ⚠️ **Preliminary — measured on battery power.** The numbers below were measured on a
+> laptop running on battery, where macOS throttles the CPU. They are **not yet authoritative**
+> and are pending a re-run on AC (mains) power before publication. Treat them as a floor, not
+> confirmed figures.
 
 **Environment:** Apple M1, 8 cores, 8 GB RAM, macOS (battery power) · via `mq-bridge-app` 0.2.6 · postgres:16
 
-### Validated run — bulk copy Postgres → JSONL
+### Preliminary run — bulk copy Postgres → JSONL
 
 | Scenario | Payload | Batch | Concurrency | Source → Sink | Result |
 | --- | --- | --- | --- | --- | --- |
@@ -76,27 +76,27 @@ theirs. The run brief lives in [`mq-bridge-app-benchmark-prompt.md`](mq-bridge-a
 
 Command: `mq-bridge-app copy --from postgres://…?table=…&cursor_column=id --to file://…?format=raw --drain --batch-size 1024 --concurrency 4`, wall-clocked. Requires an index on the cursor column (`CREATE INDEX ON <table>(id)`) — without it the keyset-pagination reader (`WHERE id > $cursor ORDER BY id LIMIT batch`) does a full scan per batch (near-quadratic).
 
-### Validated run — Postgres → JSONL vs. Meltano
+### Preliminary run — Postgres → JSONL vs. Meltano
 
-Same source table, same Postgres instance, same machine, both one-shot full-table syncs to a local JSONL file — mq-bridge-app's `copy` CLI vs. Meltano's `tap-postgres` → `target-jsonl` (median of 5 timed runs each side).
+Same source table (`bench`: 1,000,000 rows, 7 mixed-type columns), same Postgres instance, same machine, both one-shot full-table syncs to a local JSONL file — mq-bridge-app's `copy` CLI vs. Meltano's `tap-postgres` → `target-jsonl` (median of timed runs each side, row-count parity verified).
 
-| Scenario | Payload | Batch | Concurrency | Source → Sink | Result |
-| --- | --- | --- | --- | --- | --- |
-| mq-bridge-app `copy` | 256 B, 100,000 rows | 1024 | 1 | postgres (keyset cursor) → file (JSONL) | **197,628 rows/s** |
-| Meltano (`tap-postgres` → `target-jsonl`) | 256 B, 100,000 rows | default Singer config | — | postgres → JSONL | 16,330 rows/s |
+| Scenario | Payload | Batch | Concurrency | Source → Sink | Throughput | Peak RSS |
+| --- | --- | --- | --- | --- | --- | --- |
+| mq-bridge-app `copy` | 7-col, 1,000,000 rows | 1024 | 1 | postgres (keyset cursor) → file (JSONL) | **266,951 rows/s** | 19.9 MiB |
+| Meltano (`tap-postgres` → `target-jsonl`) | 7-col, 1,000,000 rows | default Singer config | — | postgres → JSONL | 15,356 rows/s | 599.7 MiB |
 
-**~12.1x faster** than Meltano in this scenario. Full setup (including the Meltano project config) is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#5--postgres--jsonl-vs-meltano-tap-postgres--target-jsonl).
+**~17.4x faster and ~30x leaner in peak memory** than Meltano in this scenario. Full setup (including the Meltano project config) is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#5--postgres--jsonl-vs-meltano-tap-postgres--target-jsonl).
 
-### Validated run — CSV → JSONL vs. Meltano
+### Preliminary run — CSV → JSONL vs. Meltano
 
 Same seeded dataset both sides, same machine, one-shot full-file CSV → local JSONL (one JSON object per input row) — mq-bridge-app's `copy` CLI vs. Meltano's `tap-csv` → `target-jsonl`.
 
-| Scenario | Payload | Batch | Concurrency | Source → Sink | Result |
-| --- | --- | --- | --- | --- | --- |
-| mq-bridge-app `copy` | 7-col mixed-type, 1,000,000 rows (~116 MiB) | 1024 | 1 | file (CSV) → file (JSONL) | **833,333 rows/s** |
-| Meltano (`tap-csv` → `target-jsonl`) | 7-col mixed-type, 1,000,000 rows | default Singer config | — | file (CSV) → JSONL | ~19,500 rows/s |
+| Scenario | Payload | Batch | Concurrency | Source → Sink | Throughput | Peak RSS |
+| --- | --- | --- | --- | --- | --- | --- |
+| mq-bridge-app `copy` | 7-col mixed-type, 1,000,000 rows (~116 MiB) | 1024 | 1 | file (CSV) → file (JSONL) | **833,333 rows/s** | 20.0 MiB |
+| Meltano (`tap-csv` → `target-jsonl`) | 7-col mixed-type, 1,000,000 rows | default Singer config | — | file (CSV) → JSONL | ~19,500 rows/s | 443.8 MiB |
 
-**~43x faster** than Meltano in this scenario. Full setup is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#6--csv--jsonl-vs-meltano-faucet-streams-headline-benchmark).
+**~43x faster and ~22x leaner in peak memory** than Meltano in this scenario. Full setup is in [mq-bridge-app's `benches/etl/README.md`](https://github.com/marcomq/mq-bridge-app/blob/dev/benches/etl/README.md#6--csv--jsonl-vs-meltano).
 
 ## Status
 

{mq_bridge_py_basic-0.3.4 → mq_bridge_py_basic-0.3.5}/benches/performance_bench.rs

@@ -360,7 +360,7 @@ pub mod aws_helper {
 pub mod zeromq_helper {
     use super::PERF_TEST_MESSAGE_COUNT;
     use mq_bridge::endpoints::zeromq::{ZeroMqConsumer, ZeroMqPublisher};
-    use mq_bridge::models::{ZeroMqConfig, ZeroMqSocketType};
+    use mq_bridge::models::{ZeroMqConfig, ZeroMqFormat, ZeroMqSocketType};
     use mq_bridge::traits::{MessageConsumer, MessagePublisher};
     use once_cell::sync::Lazy;
     use rand::RngExt;
@@ -381,6 +381,7 @@ pub mod zeromq_helper {
             bind: false,
             internal_buffer_size: Some(PERF_TEST_MESSAGE_COUNT + 1),
             topic: None,
+            format: ZeroMqFormat::Json,
         };
         Arc::new(ZeroMqPublisher::new(&config).await.unwrap())
     }
@@ -395,6 +396,7 @@ pub mod zeromq_helper {
             bind: true,
             internal_buffer_size: Some(PERF_TEST_MESSAGE_COUNT + 1),
             topic: None,
+            format: ZeroMqFormat::Json,
         };
         Arc::new(Mutex::new(ZeroMqConsumer::new(&config).await.unwrap()))
     }
@@ -595,83 +597,88 @@ pub mod file_delete_helper {
 #[cfg(feature = "http")]
 pub mod http_helper {
     use mq_bridge::endpoints::http::{HttpConsumer, HttpPublisher};
-    use mq_bridge::endpoints::memory::{MemoryConsumer, MemoryPublisher};
     use mq_bridge::models::HttpConfig;
     use mq_bridge::traits::{MessageConsumer, MessageDisposition, MessagePublisher};
-    use once_cell::sync::Lazy;
     use std::sync::Arc;
-    use std::sync::Mutex as StdMutex;
-    use tokio::sync::Mutex;
-
-    static CURRENT_URL: Lazy<StdMutex<String>> = Lazy::new(|| StdMutex::new(String::new()));
 
-    pub async fn create_consumer() -> Arc<Mutex<dyn MessageConsumer>> {
+    /// HTTP is a synchronous request/reply transport, so it does not fit the generic
+    /// write-then-read harness: that fills a buffer during the write phase and drains it during
+    /// the read phase, but for HTTP the buffer is an in-process channel — so "read" measures
+    /// memory, not the transport. There is no independent read for request/reply; the POST is
+    /// both the write and the read. We therefore measure the *coupled* round-trip throughput.
+    ///
+    /// This sets up a reliable HTTP server (a 200 is returned only once the message is
+    /// committed) whose consumer is continuously drained by a background task that commits
+    /// every message. The returned publisher thus completes a full network round-trip on every
+    /// send, so timing sends yields the true HTTP throughput. Abort the returned handle to stop
+    /// the drain task, then await it (see `finish_drain`) so receive/commit failures surface.
+    #[allow(clippy::type_complexity)]
+    pub async fn setup_coupled() -> (
+        Arc<dyn MessagePublisher>,
+        tokio::task::JoinHandle<anyhow::Result<()>>,
+    ) {
         #[cfg(feature = "rustls")]
         crate::ensure_rustls_installed();
 
-        let http_config = HttpConfig {
+        let consumer_config = HttpConfig {
             url: "127.0.0.1:0".to_string(),
-            // Sufficient internal buffer to prevent backpressure during bursts
-            internal_buffer_size: Some(super::PERF_TEST_MESSAGE_COUNT * 2),
+            internal_buffer_size: Some(super::PERF_TEST_CONCURRENCY * 4),
             concurrency_limit: Some(super::PERF_TEST_CONCURRENCY * 2),
             request_timeout_ms: Some(10000),
-            fire_and_forget: false, // Reliable mode: wait for ack before HTTP response
+            fire_and_forget: false, // reliable: 200 is sent only after commit -> real round-trip
             ..Default::default()
         };
-
-        let mut http_consumer = HttpConsumer::new(&http_config)
+        let mut consumer = HttpConsumer::new(&consumer_config)
             .await
             .expect("Failed to create HttpConsumer");
-        let addr = http_consumer
+        let addr = consumer
             .bound_addr()
             .expect("HttpConsumer should have bound addr");
         let url = format!("http://{}", addr);
 
-        {
-            let mut lock = CURRENT_URL.lock().unwrap();
-            *lock = url;
-        }
-
-        // Setup an internal memory buffer to decouple the "Write" and "Read" phases of the benchmark.
-        // This allows the benchmark to finish writing all messages without deadlocking on the reader.
-        let topic = format!("http_perf_buffer_{}", fast_uuid_v7::gen_id());
-        let mem_config = mq_bridge::models::MemoryConfig {
-            topic,
-            capacity: Some(super::PERF_TEST_MESSAGE_COUNT * 10),
-            ..Default::default()
-        };
-        let mem_publisher = MemoryPublisher::new(&mem_config).unwrap();
-        let mem_consumer = MemoryConsumer::new(&mem_config).unwrap();
-
-        // Background task to bridge Http -> Memory.
-        // We only ACK the HTTP request once the message is safely accepted by the memory queue.
-        tokio::spawn(async move {
-            while let Ok(batch) = http_consumer.receive_batch(100).await {
+        // Drain + commit so every POST gets its 200 (this is what couples the round-trip).
+        // Propagate receive/commit failures instead of swallowing them, so a broken round-trip
+        // fails the benchmark rather than reporting bogus throughput.
+        let drain = tokio::spawn(async move {
+            loop {
+                let batch = consumer
+                    .receive_batch(128)
+                    .await
+                    .map_err(|e| anyhow::anyhow!("drain receive_batch failed: {e}"))?;
                 let count = batch.messages.len();
-                if count > 0 && mem_publisher.send_batch(batch.messages).await.is_ok() {
-                    let _ = (batch.commit)(vec![MessageDisposition::Ack; count]).await;
+                if count == 0 {
+                    continue;
                 }
+                (batch.commit)(vec![MessageDisposition::Ack; count])
+                    .await
+                    .map_err(|e| anyhow::anyhow!("drain commit failed: {e}"))?;
             }
         });
 
-        Arc::new(Mutex::new(mem_consumer))
+        let publisher: Arc<dyn MessagePublisher> = Arc::new(
+            HttpPublisher::new(&HttpConfig {
+                url,
+                request_timeout_ms: Some(10000),
+                pool_idle_timeout_ms: Some(1000),
+                tcp_keepalive_ms: Some(1000),
+                ..Default::default()
+            })
+            .await
+            .unwrap(),
+        );
+        (publisher, drain)
     }
 
-    pub async fn create_publisher() -> Arc<dyn MessagePublisher> {
-        #[cfg(feature = "rustls")]
-        crate::ensure_rustls_installed();
-        let url = {
-            let lock = CURRENT_URL.lock().unwrap();
-            lock.clone()
-        };
-        let config = HttpConfig {
-            url,
-            request_timeout_ms: Some(10000),
-            pool_idle_timeout_ms: Some(1000),
-            tcp_keepalive_ms: Some(1000),
-            ..Default::default()
-        };
-        Arc::new(HttpPublisher::new(&config).await.unwrap())
+    /// Stop the drain task and surface any failure. Aborts the task, then awaits it: an
+    /// expected cancellation is fine, but a task error or panic fails the benchmark.
+    pub async fn finish_drain(drain: tokio::task::JoinHandle<anyhow::Result<()>>) {
+        drain.abort();
+        match drain.await {
+            Ok(Ok(())) => {}
+            Ok(Err(e)) => panic!("http drain task failed: {e}"),
+            Err(e) if e.is_cancelled() => {}
+            Err(e) => std::panic::resume_unwind(e.into_panic()),
+        }
     }
 }
 
@@ -1127,17 +1134,68 @@ fn performance_benchmarks(c: &mut Criterion) {
         PERF_TEST_CONCURRENCY,
         std::time::Duration::from_millis(1000)
     );
-    bench_backend!(
-        "http",
-        "http",
-        http_helper,
-        group,
-        &rt,
-        &BENCH_RESULTS,
-        PERF_TEST_MESSAGE_COUNT,
-        PERF_TEST_CONCURRENCY,
-        std::time::Duration::from_millis(20)
-    );
+    // HTTP is request/reply, so it can't be measured as decoupled write-then-read like the
+    // other endpoints (its "read" would just drain an in-process buffer at memory speed).
+    // Measure the coupled round-trip throughput and report it as both write and read.
+    #[cfg(feature = "http")]
+    if mq_bridge::test_utils::should_run_benchmark("http") {
+        group.bench_function("http_batch", |b| {
+            b.to_async(&rt).iter_custom(|iters| async move {
+                let (publisher, drain) = http_helper::setup_coupled().await;
+                tokio::time::sleep(Duration::from_millis(200)).await;
+                let mut total = Duration::ZERO;
+                for _ in 0..iters {
+                    total += mq_bridge::test_utils::measure_write_performance(
+                        "http_batch",
+                        std::sync::Arc::clone(&publisher),
+                        PERF_TEST_MESSAGE_COUNT,
+                        PERF_TEST_CONCURRENCY,
+                    )
+                    .await;
+                }
+                http_helper::finish_drain(drain).await;
+                let msgs_per_sec =
+                    (iters as f64 * PERF_TEST_MESSAGE_COUNT as f64) / total.as_secs_f64();
+                let mut results = BENCH_RESULTS.lock().await;
+                let stats = results.entry("http".to_string()).or_default();
+                stats.write_performance = msgs_per_sec;
+                stats.read_performance = msgs_per_sec;
+                println!(
+                    "\nhttp batch (coupled round-trip): {:.2} msgs/sec",
+                    msgs_per_sec
+                );
+                total
+            })
+        });
+        group.bench_function("http_single", |b| {
+            b.to_async(&rt).iter_custom(|iters| async move {
+                let (publisher, drain) = http_helper::setup_coupled().await;
+                tokio::time::sleep(Duration::from_millis(200)).await;
+                let mut total = Duration::ZERO;
+                for _ in 0..iters {
+                    total += mq_bridge::test_utils::measure_single_write_performance(
+                        "http_single",
+                        std::sync::Arc::clone(&publisher),
+                        PERF_TEST_MESSAGE_COUNT,
+                        PERF_TEST_CONCURRENCY,
+                    )
+                    .await;
+                }
+                http_helper::finish_drain(drain).await;
+                let msgs_per_sec =
+                    (iters as f64 * PERF_TEST_MESSAGE_COUNT as f64) / total.as_secs_f64();
+                let mut results = BENCH_RESULTS.lock().await;
+                let stats = results.entry("http".to_string()).or_default();
+                stats.single_write_performance = msgs_per_sec;
+                stats.single_read_performance = msgs_per_sec;
+                println!(
+                    "\nhttp single (coupled round-trip): {:.2} msgs/sec",
+                    msgs_per_sec
+                );
+                total
+            })
+        });
+    }
     bench_backend!(
         "websocket",
         "websocket",