mq-bridge-py-basic 0.3.2__tar.gz → 0.3.4__tar.gz

mq_bridge_py_basic-0.3.4/.github/workflows/benchmark.yml

@@ -0,0 +1,276 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main, dev]
+  schedule:
+    # Nightly full benchmark run (04:00 UTC)
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  # Compile the bench binary once (release, --features full) and hand the
+  # executable to the shard jobs below so none of them need to recompile.
+  benchmark-build:
+    name: Build Benchmark Binary
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+      - name: Cache cargo build
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: release-full
+
+      - name: Build bench binary
+        run: |
+          set -o pipefail
+          cargo bench --bench performance_bench --features full --no-run --message-format=json | tee build-output.json
+          BIN_PATH=$(jq -r 'select(.reason == "compiler-artifact" and (.target.kind // [] | index("bench")) and .executable != null) | .executable' build-output.json | tail -n1)
+          if [ -z "$BIN_PATH" ]; then
+            echo "Could not locate performance_bench executable in cargo build output" >&2
+            exit 1
+          fi
+          cp "$BIN_PATH" ./performance_bench_bin
+
+      - name: Upload bench binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-bench-bin
+          path: performance_bench_bin
+          retention-days: 1
+
+  # Three independent shards of the same binary, filtered via the
+  # MQB_TEST_BACKEND env var that test_utils::should_run_benchmark() already
+  # supports. Each only pre-pulls the Docker images it actually needs.
+  benchmark-streams:
+    name: Benchmark (streams)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Pre-pull docker images
+        run: |
+          for f in aws kafka amqp nats mqtt_performance; do
+            docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
+          done
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "aws,kafka,amqp,nats,mqtt,ibm-mq"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_streams.txt
+          ./scripts/ci/format_bench_output.sh raw_output_streams.txt output_streams.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-streams
+          path: |
+            raw_output_streams.txt
+            output_streams.txt
+          retention-days: 1
+
+  benchmark-database:
+    name: Benchmark (database)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Pre-pull docker images
+        run: |
+          for f in mongodb postgres; do
+            docker compose -f "tests/integration/docker-compose/${f}.yml" pull -q
+          done
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "mongodb,postgres"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_database.txt
+          ./scripts/ci/format_bench_output.sh raw_output_database.txt output_database.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-database
+          path: |
+            raw_output_database.txt
+            output_database.txt
+          retention-days: 1
+
+  benchmark-dockerless:
+    name: Benchmark (dockerless)
+    needs: benchmark-build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Download bench binary
+        uses: actions/download-artifact@v4
+        with:
+          name: performance-bench-bin
+
+      - name: Run benchmarks
+        shell: bash
+        env:
+          MQB_TEST_BACKEND: "zeromq,memory,file,http,websocket,grpc"
+        run: |
+          set -o pipefail
+          chmod +x ./performance_bench_bin
+          export CARGO_TERM_COLOR=never
+          ./performance_bench_bin --bench --output-format bencher | tee raw_output_dockerless.txt
+          ./scripts/ci/format_bench_output.sh raw_output_dockerless.txt output_dockerless.txt
+
+      - name: Upload shard output
+        uses: actions/upload-artifact@v4
+        with:
+          name: bench-output-dockerless
+          path: |
+            raw_output_dockerless.txt
+            output_dockerless.txt
+          retention-days: 1
+
+  # Merge the three shard outputs back into the single combined report the
+  # gh-pages benchmark history / dashboard has always tracked.
+  benchmark-collect:
+    name: Collect Benchmark Results
+    needs: [benchmark-streams, benchmark-database, benchmark-dockerless]
+    runs-on: ubuntu-latest
+    # Serialize gh-pages pushes so overlapping runs don't race the Pages build
+    # (concurrent pushes make GitHub cancel the superseded deployment -> "failed").
+    concurrency:
+      group: gh-pages-publish
+      cancel-in-progress: false
+    permissions:
+      contents: write
+      deployments: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download shard outputs
+        uses: actions/download-artifact@v4
+        with:
+          pattern: bench-output-*
+          merge-multiple: true
+          path: bench-outputs
+
+      - name: Merge shard outputs
+        run: |
+          cat bench-outputs/raw_output_*.txt > raw_output.txt
+          cat bench-outputs/output_*.txt > output.txt
+
+      - name: Generate Throughput Summary
+        run: |
+          python3 -c '
+          import re
+          import os
+
+          data = {}
+          try:
+              with open("raw_output.txt", "r") as f:
+                  for line in f:
+                      # Match lines like: "aws single_write: 1 iters, total time 1.48s, 673.75 msgs/sec"
+                      m = re.search(r"^\s*(.*): .* ([\d\.]+) msgs/sec", line)
+                      if m:
+                          name = m.group(1).strip()
+                          rate = float(m.group(2))
+                          if name not in data:
+                              data[name] = []
+                          data[name].append(rate)
+
+              if data:
+                  with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+                      f.write("### Benchmark Throughput Results\n")
+                      f.write("| Test Case | Average msgs/sec | Samples |\n")
+                      f.write("|-----------|------------------|---------|\n")
+                      for name in sorted(data.keys()):
+                          rates = data[name]
+                          avg = sum(rates) / len(rates)
+                          f.write(f"| {name} | {avg:,.2f} | {len(rates)} |\n")
+          except Exception as e:
+              print(f"Error generating summary: {e}")
+          '
+
+      - name: Create gh-pages branch if missing
+        run: |
+          git fetch origin gh-pages || {
+            git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git config --global user.name "github-actions[bot]"
+            git checkout --orphan gh-pages
+            git rm -rf .
+            git commit --allow-empty -m "Initialize gh-pages branch"
+            git push origin gh-pages
+            git checkout ${{ github.sha }}
+          }
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Rust Benchmark
+          tool: 'cargo'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push on both main and dev so dev commits appear in the benchmark graph.
+          # This mints a gh-pages deployment per run; the next step prunes old ones.
+          auto-push: true
+          alert-threshold: '40000%'
+          comment-on-alert: true
+          fail-on-alert: false
+
+      # Keep only the most recent gh-pages deployments so the environment's
+      # deployment list doesn't grow unbounded (one is minted per push above).
+      - name: Prune old gh-pages deployments
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          KEEP: 20
+        run: |
+          gh api "repos/${{ github.repository }}/deployments?environment=github-pages&per_page=100" --paginate -q '.[].id' \
+            | tail -n +$((KEEP + 1)) \
+            | while read -r id; do
+                gh api -X POST "repos/${{ github.repository }}/deployments/$id/statuses" -f state=inactive >/dev/null 2>&1 \
+                  || echo "::warning::failed to mark deployment $id inactive; attempting delete anyway"
+                gh api -X DELETE "repos/${{ github.repository }}/deployments/$id" >/dev/null 2>&1 && echo "pruned $id"
+              done

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.4}/.github/workflows/node.yml

@@ -2,7 +2,7 @@ name: Node
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
     paths:
       - ".github/workflows/node.yml"
       - "node/mq-bridge-node/**"
@@ -23,6 +23,28 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  # Fast guard: catches package.json/package-lock.json drift (e.g. a version
+  # bump that only half-updates the lock) before it reaches the publish flow,
+  # where `npm ci` fails after all the native builds have already run.
+  lock-sync:
+    name: package-lock in sync
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: node/mq-bridge-node/package-lock.json
+
+      - name: Verify lock is in sync
+        working-directory: node/mq-bridge-node
+        run: npm ci --ignore-scripts
+
   test:
     name: Node binding smoke test
     runs-on: ubuntu-latest

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.4}/.github/workflows/publish-node.yml

@@ -59,6 +59,12 @@ jobs:
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
           echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the build.
+      - name: Sync version from Cargo workspace
+        working-directory: node/mq-bridge-node
+        run: npm run sync-version
+
       - name: Install Node dependencies
         working-directory: node/mq-bridge-node
         run: npm ci
@@ -124,6 +130,12 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the build.
+      - name: Sync version from Cargo workspace
+        working-directory: node/mq-bridge-node
+        run: npm run sync-version
+
       - name: Install Node dependencies
         working-directory: node/mq-bridge-node
         run: npm ci
@@ -170,14 +182,16 @@ jobs:
       - name: Update npm
         run: npm install -g npm@11.5.1
 
-      - name: Install Node dependencies
-        working-directory: node/mq-bridge-node
-        run: npm ci
-
+      # Regenerate package.json/lock from the Cargo workspace version (source of
+      # truth) BEFORE npm ci, so a drifted committed lock can't fail the publish.
       - name: Sync version from Cargo workspace
         working-directory: node/mq-bridge-node
         run: npm run sync-version
 
+      - name: Install Node dependencies
+        working-directory: node/mq-bridge-node
+        run: npm ci
+
       - name: Download loader into package root
         uses: actions/download-artifact@v4
         with:

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.4}/.github/workflows/publish-python.yml

@@ -94,6 +94,39 @@ jobs:
             os: linux
             target: aarch64
             args: --release --out dist --find-interpreter
+          # Reduced-feature wheels published UNDER THE `mq-bridge-py` NAME so that
+          # a single `pip install mq-bridge-py` resolves to the `full` wheel on
+          # glibc-Linux/macOS/Windows-x64 and to this `basic`-featured wheel on
+          # targets where the full C toolchain doesn't build (musl/Alpine,
+          # Windows arm64), picked automatically by platform tag.
+          # pyproject-reduced.toml keeps the `mq-bridge-py` project name but
+          # compiles the `basic` feature set (no Kafka/sqlx/grpc/IBM-MQ-static).
+          # Same aws-lc-sys-native setup as the mq-bridge-py-basic rows.
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: ubuntu-latest
+            os: linux
+            target: x86_64
+            manylinux: musllinux_1_2
+            args: --release --out dist --find-interpreter --no-default-features
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: ubuntu-24.04-arm
+            os: linux
+            target: aarch64
+            manylinux: musllinux_1_2
+            args: --release --out dist --find-interpreter --no-default-features
+          # Windows arm64 gets the reduced set too: `basic` still pulls aws-lc-sys
+          # (via amqp+aws), which compiles natively under MSVC on the native
+          # windows-11-arm runner. If aws-lc-sys ever fails to build here, drop
+          # amqp+aws from a Windows-arm-only feature set rather than reintroducing
+          # a cross-compile.
+          - package: mq-bridge-py
+            pyproject: pyproject-reduced.toml
+            runner: windows-11-arm
+            os: windows
+            target: aarch64
+            args: --release --out dist --find-interpreter --no-default-features
           - package: mq-bridge-py
             pyproject: pyproject.toml
             runner: macos-15-intel

{mq_bridge_py_basic-0.3.2 → mq_bridge_py_basic-0.3.4}/CONFIGURATION.md

@@ -24,7 +24,7 @@ kafka_to_nats:
   output:
     nats:
       url: "nats://localhost:4222"
-      subject: "orders.processed"
+      subject: "orders_stream.processed"
       stream: "orders_stream"
 
 # Route 2: HTTP Webhook to MongoDB with Middleware
@@ -139,6 +139,38 @@ For example, to set the Kafka topic for the `kafka_to_nats` route:
 ```sh
 export MQB__KAFKA_TO_NATS__INPUT__KAFKA__TOPIC="my-other-topic"
 ```
+#### Postgres CDC example
+
+```yaml
+orders_cdc:
+  input:
+    postgres_cdc:
+      url: "postgres://user:pass@localhost:5432/app"
+      publication: "orders_pub"   # CREATE PUBLICATION orders_pub FOR TABLE orders;
+      slot_name: "mqb_orders"       # created if missing (permanent slot, resumable)
+  output:
+    nats:
+      subject: "orders.changes"
+      url: "nats://localhost:4222"
+```
+
+Each change arrives as a `CanonicalMessage` whose payload is the flat row and whose `postgres.operation` metadata marks the operation — the same convention as MongoDB CDC, so typed handlers work identically across both. The replication transport uses the published [`pgwire-replication`](https://crates.io/crates/pgwire-replication) crate.
+
+### NATS JetStream Notes
+
+Two gotchas worth knowing before wiring up a `nats` endpoint:
+
+- **Subject must be prefixed with the stream name.** When mq-bridge auto-creates
+  a JetStream stream (no existing stream already covers the subject), it scopes
+  the stream to `{stream}.>`. So `stream: "orders_stream"` requires a subject
+  like `orders_stream.foo` — a subject such as `orders.foo` will fail to publish
+  with "no stream found for given subject". This only applies to
+  auto-creation; publishing to a stream that already exists with a wider
+  subject filter works regardless of naming.
+- **`stream` is required even in Core NATS mode.** Consumer validation requires
+  a `stream` value even when `no_jetstream: true`. It's unused for the actual
+  Core NATS subscribe, but validation still rejects a missing value — pass any
+  placeholder string.
 
 ### Middleware Configuration
 Middleware is defined as a list under an endpoint.
@@ -195,8 +227,10 @@ tls:
      producer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
      consumer_options: [["ssl.cipher.suites", "ECDHE-RSA-AES256-GCM-SHA384"]]
    ```
-5. **IBM MQ** (native stack): set a strong `cipher_spec` (a TLS 1.2/1.3 CipherSpec) — it is
-   required for encrypted connections.
+5. **IBM MQ** (native stack): set a strong `tls.cipher_spec` (a TLS 1.2/1.3 CipherSpec) — it
+   is required for encrypted connections. Note `cipher_spec` lives under `tls`, not at the
+   top level of the `ibmmq` config (a breaking rename from earlier releases, where it was
+   `ibmmq.cipher_spec`).
 6. **Keep sensitive payloads out of logs.** Message payloads are emitted at `trace` level;
    run production above `trace` and confirm no cardholder data (PAN) reaches logs or traces.
 7. **Do not commit secrets.** Source passwords and tokens from a secrets manager or env vars