mq-bridge-py-basic 0.3.2__tar.gz → 0.3.3__tar.gz

mq_bridge_py_basic-0.3.3/.github/workflows/benchmark.yml

@@ -0,0 +1,254 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main, dev]
+  schedule:
+    # Nightly full benchmark run (04:00 UTC)
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  # Compile the bench binary once (release, --features full) and hand the
+  # executable to the shard jobs below so none of them need to recompile.
+  benchmark-build:
+    name: Build Benchmark Binary
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: release-full
+
+      - name: Build bench binary
+        run: |
+          set -o pipefail
+          cargo bench --bench performance_bench --features full --no-run --message-format=json | tee build-output.json
+          BIN_PATH=$(jq -r 'select(.reason == "compiler-artifact" and (.target.kind // [] | index("bench")) and .executable != null) | .executable' build-output.json | tail -n1)
+          if [ -z "$BIN_PATH" ]; then
+            echo "Could not locate performance_bench executable in cargo build output" >&2
+            exit 1
+          fi
+          cp "$BIN_PATH" ./performance_bench_bin
+
+      - name: Upload bench binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-bench-bin
+          path: performance_bench_bin
+          retention-days: 1
+
+  # Three independent shards of the same binary, filtered via the
+  # MQB_TEST_BACKEND env var that test_utils::should_run_benchmark() already
+  # supports. Each only pre-pulls the Docker images it actually needs.
+  benchmark-streams:
+    name: Benchmark (streams)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Pre-pull docker images
+        run: |
+          for f in aws kafka amqp nats mqtt_performance; do
+            docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
+          done
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "aws,kafka,amqp,nats,mqtt,ibm-mq"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_streams.txt
+          ./scripts/ci/format_bench_output.sh raw_output_streams.txt output_streams.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-streams
+          path: |
+            raw_output_streams.txt
+            output_streams.txt
+          retention-days: 1
+
+  benchmark-database:
+    name: Benchmark (database)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Pre-pull docker images
+        run: |
+          for f in mongodb postgres; do
+            docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
+          done
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "mongodb,postgres"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_database.txt
+          ./scripts/ci/format_bench_output.sh raw_output_database.txt output_database.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-database
+          path: |
+            raw_output_database.txt
+            output_database.txt
+          retention-days: 1
+
+  benchmark-dockerless:
+    name: Benchmark (dockerless)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "zeromq,memory,file,http,websocket,grpc"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_dockerless.txt
+          ./scripts/ci/format_bench_output.sh raw_output_dockerless.txt output_dockerless.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-dockerless
+          path: |
+            raw_output_dockerless.txt
+            output_dockerless.txt
+          retention-days: 1
+
+  # Merge the three shard outputs back into the single combined report the
+  # gh-pages benchmark history / dashboard has always tracked.
+  benchmark-collect:
+    name: Collect Benchmark Results
+    needs: [benchmark-streams, benchmark-database, benchmark-dockerless]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      deployments: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download shard outputs
+        uses: actions/download-artifact@v4
+        with:
+          pattern: bench-output-*
+          merge-multiple: true
+          path: bench-outputs
+
+      - name: Merge shard outputs
+        run: |
+          cat bench-outputs/raw_output_*.txt > raw_output.txt
+          cat bench-outputs/output_*.txt > output.txt
+
+      - name: Generate Throughput Summary
+        run: |
+          python3 -c '
+          import re
+          import os
+
+          data = {}
+          try:
+              with open("raw_output.txt", "r") as f:
+                  for line in f:
+                      # Match lines like: "aws single_write: 1 iters, total time 1.48s, 673.75 msgs/sec"
+                      m = re.search(r"^\s*(.*): .* ([\d\.]+) msgs/sec", line)
+                      if m:
+                          name = m.group(1).strip()
+                          rate = float(m.group(2))
+                          if name not in data:
+                              data[name] = []
+                          data[name].append(rate)
+
+              if data:
+                  with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+                      f.write("### Benchmark Throughput Results\n")
+                      f.write("| Test Case | Average msgs/sec | Samples |\n")
+                      f.write("|-----------|------------------|---------|\n")
+                      for name in sorted(data.keys()):
+                          rates = data[name]
+                          avg = sum(rates) / len(rates)
+                          f.write(f"| {name} | {avg:,.2f} | {len(rates)} |\n")
+          except Exception as e:
+              print(f"Error generating summary: {e}")
+          '
+
+      - name: Create gh-pages branch if missing
+        run: |
+          git fetch origin gh-pages || {
+            git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git config --global user.name "github-actions[bot]"
+            git checkout --orphan gh-pages
+            git rm -rf .
+            git commit --allow-empty -m "Initialize gh-pages branch"
+            git push origin gh-pages
+            git checkout ${{ github.sha }}
+          }
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Rust Benchmark
+          tool: 'cargo'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: '40000%'
+          comment-on-alert: true
+          fail-on-alert: false

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/.github/workflows/node.yml

@@ -2,7 +2,7 @@ name: Node
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
     paths:
       - ".github/workflows/node.yml"
       - "node/mq-bridge-node/**"
@@ -23,6 +23,28 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  # Fast guard: catches package.json/package-lock.json drift (e.g. a version
+  # bump that only half-updates the lock) before it reaches the publish flow,
+  # where `npm ci` fails after all the native builds have already run.
+  lock-sync:
+    name: package-lock in sync
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: node/mq-bridge-node/package-lock.json
+
+      - name: Verify lock is in sync
+        working-directory: node/mq-bridge-node
+        run: npm ci --ignore-scripts
+
   test:
     name: Node binding smoke test
     runs-on: ubuntu-latest

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/.github/workflows/publish-node.yml

@@ -59,6 +59,12 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
           echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the build.
+      - name: Sync version from Cargo workspace
+        working-directory: node/mq-bridge-node
+        run: npm run sync-version
+
       - name: Install Node dependencies
         working-directory: node/mq-bridge-node
         run: npm ci
@@ -124,6 +130,12 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the build.
+      - name: Sync version from Cargo workspace
+        working-directory: node/mq-bridge-node
+        run: npm run sync-version
+
       - name: Install Node dependencies
         working-directory: node/mq-bridge-node
         run: npm ci
@@ -170,14 +182,16 @@ jobs:
       - name: Update npm
         run: npm install -g npm@11.5.1
 
-      - name: Install Node dependencies
-        working-directory: node/mq-bridge-node
-        run: npm ci
-
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the publish.
       - name: Sync version from Cargo workspace
         working-directory: node/mq-bridge-node
         run: npm run sync-version
 
+      - name: Install Node dependencies
+        working-directory: node/mq-bridge-node
+        run: npm ci
+
       - name: Download loader into package root
         uses: actions/download-artifact@v4
         with:

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/.github/workflows/publish-python.yml

@@ -94,6 +94,39 @@ jobs:
             os: linux
             target: aarch64
             args: --release --out dist --find-interpreter
+          # Reduced-feature wheels published UNDER THE `mq-bridge-py` NAME so that
+          # a single `pip install mq-bridge-py` resolves to the `full` wheel on
+          # glibc-Linux/macOS/Windows-x64 and to this `basic`-featured wheel on
+          # targets where the full C toolchain doesn't build (musl/Alpine,
+          # Windows arm64), picked automatically by platform tag.
+          # pyproject-reduced.toml keeps the `mq-bridge-py` project name but
+          # compiles the `basic` feature set (no Kafka/sqlx/grpc/IBM-MQ-static).
+          # Same aws-lc-sys-native setup as the mq-bridge-py-basic rows.
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: ubuntu-latest
+            os: linux
+            target: x86_64
+            manylinux: musllinux_1_2
+            args: --release --out dist --find-interpreter --no-default-features
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: ubuntu-24.04-arm
+            os: linux
+            target: aarch64
+            manylinux: musllinux_1_2
+            args: --release --out dist --find-interpreter --no-default-features
+          # Windows arm64 gets the reduced set too: `basic` still pulls aws-lc-sys
+          # (via amqp+aws), which compiles natively under MSVC on the native
+          # windows-11-arm runner. If aws-lc-sys ever fails to build here, drop
+          # amqp+aws from a Windows-arm-only feature set rather than reintroducing
+          # a cross-compile.
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: windows-11-arm
+            os: windows
+            target: aarch64
+            args: --release --out dist --find-interpreter --no-default-features
           - package: mq-bridge-py
             pyproject: pyproject.toml
             runner: macos-15-intel

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/CONFIGURATION.md

@@ -24,7 +24,7 @@ kafka_to_nats:
   output:
     nats:
       url: "nats://localhost:4222"
-      subject: "orders.processed"
+      subject: "orders_stream.processed"
       stream: "orders_stream"
 
 # Route 2: HTTP Webhook to MongoDB with Middleware
@@ -140,6 +140,22 @@ For example, to set the Kafka topic for the `kafka_to_nats` route:
 export MQB__KAFKA_TO_NATS__INPUT__KAFKA__TOPIC="my-other-topic"
 ```
 
+### NATS JetStream Notes
+
+Two gotchas worth knowing before wiring up a `nats` endpoint:
+
+- **Subject must be prefixed with the stream name.** When mq-bridge auto-creates
+  a JetStream stream (no existing stream already covers the subject), it scopes
+  the stream to `{stream}.>`. So `stream: "orders_stream"` requires a subject
+  like `orders_stream.foo` — a subject such as `orders.foo` will fail to publish
+  with "no stream found for given subject". This only applies to
+  auto-creation; publishing to a stream that already exists with a wider
+  subject filter works regardless of naming.
+- **`stream` is required even in Core NATS mode.** Consumer validation requires
+  a `stream` value even when `no_jetstream: true`. It's unused for the actual
+  Core NATS subscribe, but validation still rejects a missing value — pass any
+  placeholder string.
+
 ### Middleware Configuration
 Middleware is defined as a list under an endpoint.
 
@@ -195,8 +211,10 @@ tls:
      producer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
      consumer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
    ```
-5. **IBM MQ** (native stack): set a strong `cipher_spec` (a TLS 1.2/1.3 CipherSpec) — it is
-   required for encrypted connections.
+5. **IBM MQ** (native stack): set a strong `tls.cipher_spec` (a TLS 1.2/1.3 CipherSpec) — it
+   is required for encrypted connections. Note `cipher_spec` lives under `tls`, not at the
+   top level of the `ibmmq` config (a breaking rename from earlier releases, where it was
+   `ibmmq.cipher_spec`).
 6. **Keep sensitive payloads out of logs.** Message payloads are emitted at `trace` level;
    run production above `trace` and confirm no cardholder data (PAN) reaches logs or traces.
 7. **Do not commit secrets.** Source passwords and tokens from a secrets manager or env vars

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/Cargo.lock

@@ -142,6 +142,12 @@ dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "arcstr"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d"
+
 [[package]]
 name = "arraydeque"
 version = "0.5.1"
@@ -1084,7 +1090,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
 dependencies = [
  "bytes",
+ "futures-core",
  "memchr",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
 ]
 
 [[package]]
@@ -3109,7 +3119,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -3146,6 +3156,7 @@ dependencies = [
  "rand 0.10.1",
  "rcgen",
  "rdkafka",
+ "redis",
  "reqwest",
  "rmp-serde",
  "rumqttc",
@@ -3177,7 +3188,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3185,11 +3196,13 @@ dependencies = [
  "serde",
  "serde_yaml_ng",
  "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3205,11 +3218,12 @@ dependencies = [
  "serde_yaml_ng",
  "tokio",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.3.2"
+version = "0.3.3"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3225,6 +3239,7 @@ dependencies = [
  "serde_yaml_ng",
  "tokio",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -4351,6 +4366,36 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "redis"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fa6f8e4b491d7a8ef3a9550a4d71969bd0064f46e32b8dbbcc7fc60dad94fed"
+dependencies = [
+ "arc-swap",
+ "arcstr",
+ "async-lock",
+ "backon",
+ "bytes",
+ "cfg-if",
+ "combine",
+ "futures-channel",
+ "futures-util",
+ "itoa",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls 0.23.40",
+ "rustls-native-certs",
+ "ryu",
+ "socket2 0.6.4",
+ "tokio",
+ "tokio-rustls 0.26.4",
+ "tokio-util",
+ "url",
+ "webpki-roots 1.0.7",
+ "xxhash-rust",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -6805,6 +6850,12 @@ version = "0.13.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
+[[package]]
+name = "xxhash-rust"
+version = "0.8.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
+
 [[package]]
 name = "yaml-rust2"
 version = "0.11.0"

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.3}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.2"
+version = "0.3.3"
 
 [workspace.dependencies]
 anyhow = "1.0"
@@ -137,6 +137,11 @@ sqlx = { version = "0.9", features = ["runtime-tokio", "tls-rustls", "postgres",
 # ZeroMQ
 zeromq = { version = "0.6", optional = true }
 
+# Redis Streams. Pure-Rust client (no C bindings). TLS via rustls (ring), so
+# `rediss://` URLs work; webpki-roots ships the CA bundle so there's no system
+# cert dependency. redis-rs wires its own ring-backed rustls config.
+redis = { version = "1.3", default-features = false, features = ["tokio-comp", "tokio-rustls-comp", "tls-rustls-webpki-roots", "streams", "connection-manager"], optional = true }
+
 # gRPC
 tonic = { version = "0.14", optional = true, default-features = false, features = ["codegen", "transport", "tls-webpki-roots"] }
 tonic-prost = { version = "0.14", optional = true }
@@ -163,12 +168,14 @@ middleware = ["metrics", "dedup"]
 # runtime the client is loaded lazily only when an ibm-mq endpoint is opened
 # (clear error if the client isn't installed). Use `ibm-mq-static` instead for
 # static linking.
-full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "sled", "grpc", "sqlx", "websocket", "rustls-aws-lc", "ibm-mq"]
+full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "websocket", "rustls-aws-lc", "ibm-mq"]
 
 # Functionality features
 dedup = ["sled"]
 metrics = ["dep:metrics"]
 test-utils = ["dep:tracing-appender", "dep:tracing-subscriber", "dep:serde_yaml_ng"]
+# Opt-in produce/consume-only isolation benchmarks (throwaway diagnostics, off by default)
+perf-diagnostics = ["test-utils"]
 
 # TLS crypto provider selection — pick exactly one when using any TLS endpoint
 # (http, grpc, mqtt, nats, amqp, …).  If neither is enabled the application binary
@@ -197,12 +204,15 @@ aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-sns", "dep:base64"]
 # endpoint is actually used.
 # mqi/mqai is required because mqi's always-compiled (doc-hidden) test module
 # references MqaiLibrary on the dlopen2 path; it adds only pregen bindings, no SDK.
-ibm-mq = ["dep:mqi", "mqi/dlopen2", "mqi/mqai", "mqi/mqc_9_2_0_0", "dep:libmqm-sys", "libmqm-sys/dlopen2"]
-# Static-link IBM MQ client (needs the IBM SDK at build time). mqc_9_2_0_0 matches
-# the crate's previous default so behavior is unchanged. Enabling this alongside
-# `ibm-mq` (dlopen) forces static linking — the SDK is then required at build.
-ibm-mq-static = ["dep:mqi", "mqi/link", "mqi/mqc_9_2_0_0"]
+# Capability level is pinned at mqc_9_3_0_0 (9.2 is EOL); this also unlocks
+# `tls.cert_password` support for password-protected CMS key repositories.
+ibm-mq = ["dep:mqi", "mqi/dlopen2", "mqi/mqai", "mqi/mqc_9_3_0_0", "dep:libmqm-sys", "libmqm-sys/dlopen2"]
+# Static-link IBM MQ client (needs the IBM SDK at build time). Enabling this
+# alongside `ibm-mq` (dlopen) forces static linking — the SDK is then required
+# at build.
+ibm-mq-static = ["dep:mqi", "mqi/link", "mqi/mqc_9_3_0_0"]
 zeromq = ["dep:zeromq"]
+redis-streams = ["dep:redis"]
 # Note: The grpc feature requires protobuf-src which can significantly increase build times.
 grpc = ["tonic", "prost", "tokio-stream", "tonic-prost", "tonic-prost-build", "protoc-bin-vendored", "rustls"]
 sqlx = ["dep:sqlx", "url"]