mq-bridge-py-basic 0.2.21__tar.gz → 0.3.1__tar.gz

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/.github/workflows/ci.yml

@@ -23,17 +23,13 @@ jobs:
         with:
           components: rustfmt, clippy
       
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
       
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -65,24 +61,20 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
       
-      - name: Build
+      - name: Check
         run: |
           if [ -z "${{ matrix.features }}" ]; then
-            cargo build --release
+            cargo check --all-targets
           else
-            cargo build --release --features "${{ matrix.features }}"
+            cargo check --all-targets --features "${{ matrix.features }}"
           fi
 
   # Unit tests
@@ -95,24 +87,26 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
       
       - name: Run unit tests
-        run: cargo test --lib --features=full 
+        run: cargo test --lib --features=full
 # currently - just random panic test
 #      - name: Run long-running unit tests
 #        run: cargo test --lib --features=full -- --ignored --nocapture
 
+      - name: Run non-Docker integration tests
+        run: cargo test --test ref_test --test sqlite_test --test tls_example --test websocket_test --features=full
+
+      - name: Memory leak soak (route commit-task JoinSet)
+        run: cargo test --test memory_leak_test --features=full -- --ignored --nocapture
+
   # Integration tests (requires Docker) split into parallel jobs
   integration-others:
     name: Integration — Others
@@ -127,17 +121,13 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
       
       - name: Pre-pull docker images
         run: |
@@ -175,17 +165,13 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
 
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
 
       - name: Pre-pull docker images
         run: |
@@ -223,17 +209,13 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
 
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
 
       - name: Pre-pull docker images
         run: |
@@ -264,20 +246,13 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
       
-      - name: Cache cargo registry
-        uses: actions/cache@v4
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-
-      
-      - name: Build documentation
-        run: cargo doc --all-features --no-deps
+          # Differentiates the matrix `check` job per feature set; evaluates to an
+          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
+          # job id, OS, rustc version, and lockfiles automatically.
+          key: ${{ matrix.features }}
       
-      - name: Check for broken links
-        run: cargo doc --all-features --no-deps 2>&1 | grep -i "warning\|error" && exit 1 || exit 0
+      - name: Build documentation (deny warnings)
+        run: RUSTDOCFLAGS="-D warnings" cargo doc --all-features --no-deps

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/.github/workflows/publish-node.yml

@@ -10,11 +10,12 @@ name: Publish Node Package
 # RHEL 9 (glibc 2.34), Debian 11/12 and Ubuntu 20.04+ — covering the priority
 # server targets. macOS/Windows build natively.
 #
-# Auth: token-or-OIDC. If the `NPM_TOKEN` secret is set it is used (needed for the
-# one-time bootstrap that creates the 5 platform packages, since npm has no
-# pending-publisher concept). Once all six packages exist and trusted publishing
-# is configured for them, remove the secret and publishing continues via OIDC
-# (id-token) with provenance — no token.
+# Auth: OIDC trusted publishing. The one-time bootstrap (which used an NPM_TOKEN
+# secret to create the 5 platform packages, since npm has no pending-publisher
+# concept) is done; all six packages exist with trusted publishing configured, so
+# publishing runs via OIDC (id-token) with provenance — no token. We deliberately
+# do NOT read NPM_TOKEN: a present-but-expired secret would be sent to the registry
+# and 401 instead of falling back to OIDC, so the credential is left out entirely.
 
 on:
   release:
@@ -67,6 +68,9 @@ jobs:
         env:
           CARGO_NET_RETRY: "5"
           CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+          # curl's HTTP/2 multiplexing flakes on Windows runners ("[55] Failed
+          # sending data to the peer"); force HTTP/1.1 so fetches are reliable.
+          CARGO_HTTP_MULTIPLEXING: "false"
         run: >-
           npx napi build --platform --release --strip
           --target ${{ matrix.target }}
@@ -129,6 +133,9 @@ jobs:
         env:
           CARGO_NET_RETRY: "5"
           CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+          # curl's HTTP/2 multiplexing flakes on Windows runners ("[55] Failed
+          # sending data to the peer"); force HTTP/1.1 so fetches are reliable.
+          CARGO_HTTP_MULTIPLEXING: "false"
         run: >-
           npx napi build --platform --release --strip
           --target ${{ matrix.target }}
@@ -195,17 +202,56 @@ jobs:
         working-directory: node/mq-bridge-node
         run: ls -R npm
 
+      # The default per-platform README is ~88 bytes; paired with a 50-67MB
+      # binary that low-content signature trips npm's new-package spam filter
+      # (E403 "Package name triggered spam detection"). Give each stub a real
+      # README plus description/homepage/keywords so it doesn't read as a
+      # placeholder. (Server-side heuristic; reduces but cannot guarantee a pass.)
+      - name: Enrich platform package metadata
+        working-directory: node/mq-bridge-node
+        run: |
+          set -e
+          for dir in npm/*/; do
+            name=$(node -p "require('./$dir/package.json').name")
+            cat > "$dir/README.md" <<EOF
+          # $name
+
+          Prebuilt native binary for the [\`mq-bridge\`](https://github.com/marcomq/mq-bridge)
+          Node.js addon (napi-rs). This is a platform-specific dependency of the
+          \`mq-bridge\` package and is selected automatically via optionalDependencies —
+          you install \`mq-bridge\`, not this package directly.
+          EOF
+            node -e "const p='./$dir/package.json',j=require(p);j.description='Prebuilt '+j.name+' binary for the mq-bridge Node.js addon';j.homepage='https://github.com/marcomq/mq-bridge#readme';j.keywords=['mq-bridge','napi','native','prebuilt','addon'];require('fs').writeFileSync(p,JSON.stringify(j,null,2)+'\n')"
+          done
+
       - name: Publish platform packages and root
         working-directory: node/mq-bridge-node
         env:
-          # Empty when the secret is unset -> npm falls back to OIDC.
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          # No NODE_AUTH_TOKEN: publishing is OIDC-only (trusted publishing).
           NPM_CONFIG_PROVENANCE: "true"
         run: |
           set -e
-          for dir in npm/*/; do
-            echo "Publishing $dir"
+          publish() {
+            local dir="$1" name ver
+            name=$(node -p "require('./$dir/package.json').name")
+            ver=$(node -p "require('./$dir/package.json').version")
+            # Skip if this exact version is already on the registry; a re-run
+            # after a partial publish must not fail on already-published dirs.
+            if npm view "$name@$ver" version >/dev/null 2>&1; then
+              echo "Skipping $name@$ver (already published)"
+              return 0
+            fi
+            echo "Publishing $dir ($name@$ver)"
             npm publish "$dir" --access public
+          }
+          for dir in npm/*/; do
+            publish "$dir"
           done
           echo "Publishing root package"
-          npm publish --access public
+          root_name=$(node -p "require('./package.json').name")
+          root_ver=$(node -p "require('./package.json').version")
+          if npm view "$root_name@$root_ver" version >/dev/null 2>&1; then
+            echo "Skipping $root_name@$root_ver (already published)"
+          else
+            npm publish --access public
+          fi

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/CONFIGURATION.md

@@ -161,6 +161,56 @@ input:
     # ... kafka config
 ```
 
+### TLS & Security Hardening
+
+Most endpoints accept a `tls` block. The available fields are:
+
+```yaml
+tls:
+  required: true                   # enable TLS
+  ca_file: "./certs/ca.pem"        # CA to verify the server
+  cert_file: "./certs/client.pem"  # client cert (mTLS)
+  key_file: "./certs/client.key"   # client private key (mTLS)
+  cert_password: "secret"          # password for an encrypted key (where supported)
+  accept_invalid_certs: false      # NEVER set true in production
+```
+
+**Hardening checklist (e.g. for PCI-DSS Req 4.2.1):**
+
+1. **Enable TLS on every endpoint carrying sensitive data** (`required: true`) and supply
+   a `ca_file`. Use mTLS (`cert_file` + `key_file`) for mutual authentication where the
+   broker supports it.
+2. **Never disable certificate validation.** `accept_invalid_certs` defaults to `false`;
+   leaving it that way is required — setting it `true` on a sensitive path defeats TLS.
+3. **Choose the crypto provider feature.** Build with `rustls-aws-lc` (FIPS-capable, also
+   enables post-quantum key exchange) or `rustls-ring`. The rustls-based endpoints — NATS,
+   MQTT, HTTP, gRPC, WebSocket, AMQP — only ever negotiate rustls's safe TLS 1.2/1.3 AEAD
+   cipher suites; weak/legacy suites (RC4, 3DES, CBC-SHA1, export) cannot be offered, so
+   "strong ciphers only" holds without any explicit cipher list.
+4. **Kafka** (librdkafka/OpenSSL): certificate verification is on by default
+   (`enable.ssl.certificate.verification` follows `accept_invalid_certs`). If an auditor
+   requires an explicit allowlist, pin it via `producer_options` / `consumer_options`:
+   ```yaml
+   kafka:
+     producer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
+     consumer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
+   ```
+5. **IBM MQ** (native stack): set a strong `cipher_spec` (a TLS 1.2/1.3 CipherSpec) — it is
+   required for encrypted connections.
+6. **Keep sensitive payloads out of logs.** Message payloads are emitted at `trace` level;
+   run production above `trace` and confirm no cardholder data (PAN) reaches logs or traces.
+7. **Do not commit secrets.** Source passwords and tokens from a secrets manager or env vars
+   (`MQB__...`) rather than checked-in config.
+
+**Notes and boundaries:**
+
+- TLS 1.3 alone is sufficient in 2026 (Mozilla "Modern" profile). The rustls endpoints
+  currently negotiate **TLS 1.2 and 1.3** (both are PCI-acceptable). A central
+  "TLS 1.3-only" toggle is not yet configurable in the library; enforce a minimum protocol
+  version on the broker/server side, which is the side that accepts the connection.
+- Kafka and IBM MQ use native TLS stacks, so a library-wide version policy cannot be
+  applied to them — configure their minimum TLS version on the broker.
+
 ### HTTP Consumer Fast Path
 
 Compatible `http -> response` routes may use an inline response fast path for lower latency. This bypasses the normal route consumer/worker/disposition pipeline, but it still keeps the output publisher chain active, including output handlers and allowed output middlewares.
@@ -183,6 +233,36 @@ input:
 
 This is useful when you want stable, explicit semantics regardless of future optimizations, or when you want to avoid the inline path's response behavior differences. In particular, the inline path does not automatically echo unchanged request metadata back as HTTP response headers.
 
+### Connection Sharing
+
+Publishers that target the same server reuse one underlying transport client by
+default, instead of each opening its own. This consolidates TCP connections, background
+threads, and batching, and follows each driver's own guidance (one shared producer /
+client / pool per application). Sharing applies to **Kafka, NATS, MongoDB, SQLx, HTTP,
+and gRPC**; the client is keyed by its connection-level settings (URL, auth, TLS, and
+client-level options), never by topic/subject/collection. A shared client is released
+once the last publisher using it is dropped.
+
+Set `shared: false` on a publisher to give it a dedicated connection:
+
+```yaml
+orders_out:
+  output:
+    kafka:
+      topic: "orders"
+      url: "localhost:9092"
+      shared: false   # dedicated producer — keeps this latency-sensitive topic off a busy producer's queue
+```
+
+*   **Kafka**: a single producer serves every topic and is the recommended setup. Use
+    `shared: false` to isolate a latency-sensitive topic from a high-throughput one so
+    they don't share one internal send queue (head-of-line blocking).
+*   **SQLx**: a shared pool means its `max_connections` is a budget shared across every
+    route using that database. Use `shared: false` if a route needs its own pool.
+*   **gRPC**: a shared channel multiplexes over one HTTP/2 connection; at very high
+    concurrency its max-concurrent-streams cap can bottleneck — `shared: false` gives a
+    dedicated channel.
+
 ### Specialized Endpoints
 
 #### Switch

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/Cargo.lock

@@ -3082,7 +3082,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.2.21"
+version = "0.3.1"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -3147,20 +3147,32 @@ dependencies = [
  "zeromq",
 ]
 
+[[package]]
+name = "mq-bridge-bindings-common"
+version = "0.3.1"
+dependencies = [
+ "anyhow",
+ "fast-uuid-v7",
+ "mq-bridge",
+ "serde",
+ "serde_yaml_ng",
+ "tokio",
+]
+
 [[package]]
 name = "mq-bridge-node"
-version = "0.2.21"
+version = "0.3.1"
 dependencies = [
  "anyhow",
  "async-trait",
  "bytes",
  "fast-uuid-v7",
  "mq-bridge",
+ "mq-bridge-bindings-common",
  "napi",
  "napi-build",
  "napi-derive",
  "schemars",
- "serde",
  "serde_json",
  "serde_yaml_ng",
  "tokio",
@@ -3169,13 +3181,14 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.2.21"
+version = "0.3.1"
 dependencies = [
  "anyhow",
  "async-trait",
  "bytes",
  "fast-uuid-v7",
  "mq-bridge",
+ "mq-bridge-bindings-common",
  "pyo3",
  "rustls 0.23.40",
  "schemars",

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/Cargo.toml

@@ -12,11 +12,11 @@ keywords = ["messaging", "kafka", "nats", "amqp", "cqrs"]
 categories = ["asynchronous", "network-programming"]
 
 [workspace]
-members = ["python/mq-bridge-py"]
+members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.2.21"
+version = "0.3.1"
 
 [workspace.dependencies]
 anyhow = "1.0"

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.2.21
+Version: 0.3.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -38,12 +38,14 @@ Memory and file endpoints are always present in both packages. Use `mq-bridge-py
 
 The public API stays close to mq-bridge itself:
 
-- `Route.from_yaml(path, name)` loads one named route from a YAML file
-- `Route.from_yaml_str(text, name)` / `Route.from_config(mapping, name)` build a route from an in-memory YAML string or a Python `dict`, no file required
+- `Route.from_file(path, name=None)` loads a route from a YAML/JSON file. The three constructors differ only by source: `from_file` (path), `from_str` (in-memory YAML/JSON string), `from_config` (Python `dict`)
+- The `name` is optional: pass it to pick one entry out of a `routes:`/`publishers:` document, or omit it to treat the whole config as a single bare route/endpoint body
 - `Route.with_handler(...)` attaches a raw `Message` handler, with lazy `json()`/`text()` readers and `with_json()`/`with_payload()` response helpers
 - `Route.add_handler(kind, ...)` uses mq-bridge's `kind` dispatch and delivers decoded JSON
 - `RetryableError` and `NonRetryableError` let Python handlers signal retry intent
-- `Publisher.from_yaml(path, name)` (plus `from_yaml_str` / `from_config`) loads one named publisher
+- `Publisher.from_file(path, name=None)` (plus `from_str` / `from_config`) builds a publisher endpoint
+
+`from_yaml` / `from_yaml_str` remain as deprecated aliases for `from_file` / `from_str`.
 - `Publisher.send_json(...)` and `Publisher.request_json(...)` serialize Python JSON values in Rust
 
 The Python surface is synchronous and blocking. Tokio, broker I/O, routing, and batching all stay in Rust.
@@ -53,10 +55,10 @@ The Python surface is synchronous and blocking. Tokio, broker I/O, routing, and
 `mq-bridge-app` can create and test route and endpoint JSON/YAML through its UI.
 It does not replace your Python code or handlers, but it is useful when you want
 a known-good connection and route shape before pasting the configuration into
-Python. Load the generated config with `Route.from_config`, `Route.from_yaml`,
-`Publisher.from_config`, or `Publisher.from_yaml`.
+Python. Load the generated config with `Route.from_config`, `Route.from_file`,
+`Publisher.from_config`, or `Publisher.from_file`.
 
-For the `from_config` / `from_yaml_str` mappings, `mq_bridge.config` ships
+For the `from_config` / `from_str` mappings, `mq_bridge.config` ships
 `TypedDict` definitions so editors autocomplete the config keys (`input`,
 `output`, `batch_size`, every transport config, middleware, …):
 
@@ -117,6 +119,91 @@ with Route.from_config(config, "orders_route").with_handler(handle):
 Configuration/connection errors surface from `start()` itself, not from a
 background thread. `run()` remains available for the blocking single-route case.
 
+## Pull-based consumer
+
+`Route` is push-based: you attach a handler and the route drives it. When you
+instead want to **pull** messages on your own schedule — e.g. to feed a
+generator-style sink such as a [`dlt`](https://dlthub.com) resource — use
+`Consumer`. It wraps any input endpoint and hands batches back to Python:
+
+```python
+from mq_bridge import Consumer
+
+consumer = Consumer.from_config({"nats": {"subject": "orders", "url": "nats://localhost:4222"}})
+
+while not consumer.exhausted:
+    batch = consumer.poll(max=500, timeout_ms=1000)   # [] on timeout
+    if not batch:
+        continue
+    for message in batch:
+        handle(message.json())
+    consumer.commit()                                 # ack only after handling
+```
+
+`poll()` receives up to `max` messages **without** acknowledging them;
+`commit()` acks every batch returned since the last commit, advancing the
+consumer offset (or removing them from the queue). Committing only after the
+downstream write succeeds gives at-least-once delivery: a crash before `commit()`
+re-delivers the batch. `poll()` returns `[]` once `timeout_ms` elapses with
+nothing received (omit it to block until a message arrives), and sets
+`exhausted` once a bounded source (e.g. a file) is fully drained — streaming
+brokers never set it.
+
+> **You must call `commit()` — it is not optional.** It is the only thing that
+> tells the broker a batch is done. If you keep polling without committing:
+> - the consumer offset never advances, so every message is **re-delivered** on
+>   the next run (and you reprocess from the start);
+> - most brokers stop sending once their unacknowledged/prefetch window fills, so
+>   `poll()` eventually **stalls** and returns nothing;
+> - the uncommitted batches are held in memory pending their ack, so the process
+>   **grows unbounded**.
+>
+> Commit after each batch you have durably handled (as in the loops above). If a
+> batch fails downstream, simply *don't* commit it — it will be redelivered.
+
+`consumer.status()` returns a snapshot dict (`healthy`, `target`, `pending`,
+`capacity`, `error`, `details`). `pending` is the broker backlog/lag where the
+transport reports it — Kafka offset lag, AMQP queue depth, NATS JetStream
+`num_pending` — so `pending == 0` is a precise "caught up" check for a bounded
+drain; it is `None` where the broker exposes no backlog (core NATS, MQTT), where
+you fall back to a `timeout_ms` that returns `[]`. It's a point-in-time snapshot,
+not a guarantee.
+
+`consumer.close()` releases the broker connection; it's idempotent, and `poll()`
+/`status()` raise afterwards. Python is garbage-collected, so close explicitly
+(or use the context-manager form, which closes on exit) rather than relying on
+the object being collected:
+
+```python
+with Consumer.from_config(cfg) as consumer:
+    batch = consumer.poll(max=500, timeout_ms=1000)
+    ...
+    consumer.commit()
+# connection released here
+```
+
+The endpoint config decides durability exactly as a route input does: a
+consumer-group config resumes from the last commit, a subscriber config receives
+only new messages. `Consumer.from_file` / `from_str` accept the same shapes,
+plus a named entry under a `consumers:` document section.
+
+As a `dlt` resource this is a few lines:
+
+```python
+import dlt
+from mq_bridge import Consumer
+
+@dlt.resource(name="orders")
+def orders():
+    consumer = Consumer.from_config({"nats": {"subject": "orders", "url": "nats://localhost:4222"}})
+    while not consumer.exhausted:
+        batch = consumer.poll(max=500, timeout_ms=1000)
+        if not batch:
+            break                 # nothing more pending this run
+        yield [m.json() for m in batch]
+        consumer.commit()
+```
+
 ## Tuning (environment variables)
 
 These knobs are read from the environment at startup:

{mq_bridge_py_basic-0.2.21 → mq_bridge_py_basic-0.3.1}/README.md

@@ -20,6 +20,28 @@
 It is not only a forwarder. A route can transform, filter, fan out, retry, rate-limit, deduplicate, or turn a request into a response before the message reaches the next system. The core is built on Tokio and keeps the transport details at the edge, so application code can mostly work with `CanonicalMessage`s and handlers.
 
 
+## Language Bindings
+
+`mq-bridge` is a Rust library, but the same engine ships as native bindings for **Python** and **Node.js**. The Tokio runtime, broker I/O, routing, and batching all stay in Rust; the binding is a thin layer for handlers and configuration.
+
+| Language | Package | Install |
+| :--- | :--- | :--- |
+| Rust | [`mq-bridge`](https://crates.io/crates/mq-bridge) | `cargo add mq-bridge` |
+| Python | [`mq-bridge-py`](python/mq-bridge-py/README.md) ([PyPI](https://pypi.org/project/mq-bridge-py/)) | `pip install mq-bridge-py` |
+| Node.js | [`mq-bridge`](node/mq-bridge-node/README.md) ([npm](https://www.npmjs.com/package/mq-bridge)) | `npm install mq-bridge` |
+
+The constructor names are kept aligned across languages, so a config loader reads the same in either binding (Python uses `snake_case`, Node uses `camelCase`):
+
+- `Route.from_file` / `Route.fromFile` — load a route from a YAML/JSON file
+- `Route.from_str` / `Route.fromStr` — load from an in-memory YAML/JSON string
+- `Route.from_config` / `Route.fromConfig` — load from a dict / JS object
+- the matching `Publisher.*` constructors build a publisher endpoint
+
+The `name` argument is optional in both: pass it to select one entry from a `routes:`/`publishers:` document, or omit it to treat the config as a single bare route/endpoint body.
+
+The Python binding also holds up well under load on the third-party [http-arena.com](https://www.http-arena.com/#sort=rps:-1&q=python) requests-per-second HTTP benchmark (live leaderboard — rankings shift over time). See [the Python analysis notes](python/mq-bridge-py/README.md#analysis) for the local HTTP comparison harness.
+
+
 ## Architecture
 
 See [ARCHITECTURE.md](ARCHITECTURE.md) for a detailed overview of the internal design, extensibility, and usage patterns.
@@ -55,7 +77,7 @@ That means `mq-bridge` tries to keep the boring parts boring. Kafka offsets, Rab
 
 Batching is a big part of that design. Every endpoint is optimized around batch-shaped APIs, even when the backend itself only has a single-message primitive. Batching is disabled by default (`batch_size: 1`) because it is the safest behavior and easiest to reason about. When throughput matters, increasing `batch_size` is usually the first knob to try. For example via `batch_size: 128` in yaml or `.with_batch_size(128)` for routes.
 
-The error handling follows the same idea. Batch publishing can report partial success, retryable failures, and non-retryable failures. Route commits are sequenced so cumulative-ack brokers do not accidentally acknowledge later messages before earlier batches are resolved. In other words: batching is not just a performance trick bolted onto the side; ack/nack behavior and retry/DLQ handling were built to work with it.
+The error handling follows the same idea. Batch publishing can report partial success, retryable failures, and non-retryable failures. Route commits are sequenced for cumulative-ack brokers so later batches cannot acknowledge earlier unresolved messages; transports with independent acknowledgements can commit batches concurrently. In other words: batching is not just a performance trick bolted onto the side; ack/nack behavior and retry/DLQ handling were built to work with it.
 
 What it does not try to be: a domain framework, an actor runtime, or a full stream processor. You can build CQRS-ish flows with it, but the library cares more about transport, routing, and delivery behavior than about prescribing your domain model.
 
@@ -161,7 +183,7 @@ https://github.com/marcomq/mq-bridge-app
 
 `mq-bridge-app` can be used to create and test route and endpoint configurations through a UI. The generated JSON/YAML can then be copied into an application and loaded by the Rust library or the Python bindings.
 
-This does not replace application code or handlers, but it is useful when you want a known-good connection and route shape before pasting the configuration into code. For Python projects, routes and publishers are commonly loaded from JSON/YAML with `Route.from_config`, `Route.from_yaml`, `Publisher.from_config`, or `Publisher.from_yaml`.
+This does not replace application code or handlers, but it is useful when you want a known-good connection and route shape before pasting the configuration into code. For Python and Node projects, routes and publishers are commonly loaded from JSON/YAML with `Route.from_file`/`fromFile` (a file path) and `Route.from_str`/`fromStr` (a JSON/YAML string), while `Route.from_config`/`fromConfig` takes an already-parsed dict/JS object. The matching `Publisher.*` constructors work the same way.
 
 ### Programmatic Handlers
 
@@ -392,7 +414,7 @@ Important route-level knobs:
 
 *   `batch_size`: maximum messages per route iteration. Defaults to `1`; increase it when throughput matters.
 *   `concurrency`: number of route workers. Defaults to `1`; useful for high-latency handlers or endpoints.
-*   `commit_concurrency_limit`: maximum queued in-flight commit operations used by ordered commit sequencing. Defaults to `4096`.
+*   `commit_concurrency_limit`: maximum in-flight commit operations, whether they are queued through ordered commit sequencing or run concurrently for independent-ack transports. Defaults to `4096`.
 
 Middleware can be attached to inputs or outputs. The most commonly used ones are `retry`, `dlq`, `deduplication`, `limiter`, and `cookie_jar`. Retry/DLQ are especially useful with batching because partial failures can be retried or sent to a DLQ without treating the entire batch as equally broken.