mq-bridge-py-basic 0.4.3__tar.gz → 0.4.5__tar.gz

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/.github/workflows/ci.yml

@@ -1,10 +1,20 @@
 name: CI
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main, dev]
+  workflow_dispatch:
+
+# Only the tip of a PR branch is worth testing; superseded runs are cancelled so
+# they stop holding runner slots the integration jobs are waiting for.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# Nothing in CI writes back to the repository; artifacts move through the Actions
+# runtime token, not this one.
+permissions:
+  contents: read
 
 env:
   CARGO_TERM_COLOR: always
@@ -25,11 +35,6 @@ jobs:
       
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
-        with:
-          # Differentiates the matrix `check` job per feature set; evaluates to an
-          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
-          # job id, OS, rustc version, and lockfiles automatically.
-          key: ${{ matrix.features }}
       
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -37,44 +42,37 @@ jobs:
       - name: Run clippy
         run: cargo clippy --all-targets --all-features -- -D warnings
 
-  # Build with all features
+  # Every feature subset still gets its own `cargo check`, but sequentially in one
+  # job: the matrix cost 8 runner slots at the moment integration-build's dependents
+  # were starving, and none of this is on the critical path.
   build:
-    name: Build
+    name: Build (feature subsets)
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        features:
-          - ""
-          - "full"
-          - "kafka"
-          #  "amqp" # no recent changes, already included in full
-          - "nats"
-          - "grpc"
-          - "mqtt"
-          - "mongodb"
-          - "http"
-          # - "aws"  # needs too long, already included in full
-          # - "zeromq" # no recent changes, already included in full
     steps:
       - uses: actions/checkout@v4
-      
+
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      
+
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
-        with:
-          # Differentiates the matrix `check` job per feature set; evaluates to an
-          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
-          # job id, OS, rustc version, and lockfiles automatically.
-          key: ${{ matrix.features }}
-      
-      - name: Check
+
+      # amqp/aws/zeromq are omitted: unchanged for a while and covered by `full`.
+      - name: Check each feature subset
         run: |
-          if [ -z "${{ matrix.features }}" ]; then
-            cargo check --all-targets
-          else
-            cargo check --all-targets --features "${{ matrix.features }}"
+          failed=""
+          for f in "" full kafka nats grpc mqtt mongodb http; do
+            echo "::group::cargo check --features ${f:-<default>}"
+            if [ -z "$f" ]; then
+              cargo check --all-targets || failed="$failed <default>"
+            else
+              cargo check --all-targets --features "$f" || failed="$failed $f"
+            fi
+            echo "::endgroup::"
+          done
+          if [ -n "$failed" ]; then
+            echo "::error::feature subsets failed:$failed"
+            exit 1
           fi
 
   # Unit tests
@@ -89,11 +87,6 @@ jobs:
       
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
-        with:
-          # Differentiates the matrix `check` job per feature set; evaluates to an
-          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
-          # job id, OS, rustc version, and lockfiles automatically.
-          key: ${{ matrix.features }}
       
       - name: Run unit tests
         run: cargo test --lib --features=full
@@ -145,12 +138,32 @@ jobs:
           path: nextest-archive.tar.zst
           retention-days: 1
 
-  # Integration tests (requires Docker) split into parallel jobs. Each runs the
-  # prebuilt nextest archive from integration-build — no recompilation.
-  integration-others:
-    name: Integration — Others
+  # Integration tests (requires Docker), split across runners by measured runtime
+  # so no single job dominates the critical path. Each runs the prebuilt nextest
+  # archive from integration-build — no recompilation. `--no-capture` implies
+  # `--test-threads=1`, and tests tear their compose stacks down with `down -v`,
+  # so parallelism has to come from separate runners rather than from nextest.
+  integration:
+    name: Integration — ${{ matrix.name }}
     runs-on: ubuntu-latest
     needs: integration-build
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: Status
+            filter: 'binary(integration_test) & test(=test_all_status)'
+          - name: Chaos
+            filter: 'binary(integration_test) & test(=test_all_chaos)'
+          - name: Logic
+            filter: 'binary(integration_test) & (test(=test_all_request_reply) | test(=test_all_subscriber_logic))'
+          - name: Endpoints
+            filter: 'binary(integration_test) & not test(=test_all_status) & not test(=test_all_chaos) & not test(=test_all_request_reply) & not test(=test_all_subscriber_logic) & not test(=test_all_performance_pipeline) & not test(=test_all_performance_direct)'
+          - name: Performance Pipeline
+            filter: 'binary(integration_test) & test(=test_all_performance_pipeline)'
+          - name: Performance Direct
+            filter: 'binary(integration_test) & test(=test_all_performance_direct)'
     services:
       docker:
         image: docker:dind
@@ -171,58 +184,17 @@ jobs:
       - name: Pre-pull docker images
         run: |
           find tests/integration/docker-compose -name "*.yml" | xargs -P 5 -I {} docker compose -f {} pull -q
-      - name: Install JRE (for keytool)
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y default-jre-headless
-
-      - name: Generate TLS certs for integration services
-        run: |
-          chmod +x tests/integration/scripts/gen_certs.sh
-          ./tests/integration/scripts/gen_certs.sh mongodb
-          ./tests/integration/scripts/gen_certs.sh kafka
-          ./tests/integration/scripts/gen_certs.sh ibm-mq
-
-      - name: Run chaos/stability tests
-        run: |
-          cargo nextest run --archive-file nextest-archive.tar.zst \
-            --run-ignored all --no-capture \
-            -E 'binary(integration_test) & test(=test_all_chaos)'
-
-      - name: Run remaining integration tests (excluding chaos & performance)
-        run: |
-          cargo nextest run --archive-file nextest-archive.tar.zst \
-            --run-ignored all --no-capture \
-            -E 'binary(integration_test) & not test(=test_all_chaos) & not test(=test_all_performance_pipeline) & not test(=test_all_performance_direct)'
-
-  integration-performance:
-    name: Integration — Performance
-    runs-on: ubuntu-latest
-    needs: integration-build
-    services:
-      docker:
-        image: docker:dind
-        options: --privileged
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install cargo-nextest
-        uses: taiki-e/install-action@v2
-        with:
-          tool: nextest
 
-      - name: Download nextest archive
-        uses: actions/download-artifact@v4
-        with:
-          name: integration-nextest-archive
-
-      - name: Pre-pull docker images
-        run: |
-          find tests/integration/docker-compose -name "*.yml" | xargs -P 5 -I {} docker compose -f {} pull -q
       - name: Install JRE (for keytool)
+        timeout-minutes: 5
         run: |
-          sudo apt-get update
-          sudo apt-get install -y default-jre-headless
+          # Runner images ship a JDK; only pay the 46 MB download if they didn't.
+          # A degraded apt mirror used to stall this step for hours.
+          if command -v keytool >/dev/null 2>&1; then exit 0; fi
+          sudo apt-get update -qq
+          sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            -o Acquire::Retries=3 -o Acquire::http::Timeout=20 \
+            default-jre-headless
 
       - name: Generate TLS certs for integration services
         run: |
@@ -231,26 +203,18 @@ jobs:
           ./tests/integration/scripts/gen_certs.sh kafka
           ./tests/integration/scripts/gen_certs.sh ibm-mq
 
-      - name: Run pipeline performance tests
-        run: |
-          cargo nextest run --archive-file nextest-archive.tar.zst \
-            --run-ignored all --no-capture \
-            -E 'binary(integration_test) & test(=test_all_performance_pipeline)'
-
-      - name: Run other performance tests
+      - name: Run ${{ matrix.name }} integration tests
         run: |
           cargo nextest run --archive-file nextest-archive.tar.zst \
             --run-ignored all --no-capture \
-            -E 'binary(integration_test) & test(=test_all_performance_direct)'
+            -E '${{ matrix.filter }}'
 
+  # Armature clones an external repo and runs its test suite: no docker stacks, no
+  # TLS certs, no keytool. It used to pay ~1.3m of setup for 1.7m of tests.
   integration-armature:
     name: Integration — Armature
     runs-on: ubuntu-latest
     needs: integration-build
-    services:
-      docker:
-        image: docker:dind
-        options: --privileged
     steps:
       - uses: actions/checkout@v4
 
@@ -264,21 +228,6 @@ jobs:
         with:
           name: integration-nextest-archive
 
-      - name: Pre-pull docker images
-        run: |
-          find tests/integration/docker-compose -name "*.yml" | xargs -P 5 -I {} docker compose -f {} pull -q
-      - name: Install JRE (for keytool)
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y default-jre-headless
-
-      - name: Generate TLS certs for integration services
-        run: |
-          chmod +x tests/integration/scripts/gen_certs.sh
-          ./tests/integration/scripts/gen_certs.sh mongodb
-          ./tests/integration/scripts/gen_certs.sh kafka
-          ./tests/integration/scripts/gen_certs.sh ibm-mq
-
       - name: Run armature integration tests
         run: |
           cargo nextest run --archive-file nextest-archive.tar.zst \
@@ -297,11 +246,6 @@ jobs:
       
       - name: Cache cargo build
         uses: Swatinem/rust-cache@v2
-        with:
-          # Differentiates the matrix `check` job per feature set; evaluates to an
-          # empty string (no-op) in non-matrix jobs. rust-cache already keys on the
-          # job id, OS, rustc version, and lockfiles automatically.
-          key: ${{ matrix.features }}
       
       - name: Build documentation (deny warnings)
         run: RUSTDOCFLAGS="-D warnings" cargo doc --all-features --no-deps

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/.github/workflows/ibm-mq.yml

@@ -21,6 +21,12 @@ on:
       - '.github/workflows/ibm-mq.yml'
   workflow_dispatch:
 
+# Superseded PR runs are cancelled; push runs always finish so every pre-merge
+# commit keeps its own verdict.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
@@ -144,9 +150,13 @@ jobs:
 
       - name: Install JDK (for keytool - TLS tests)
         if: steps.mqclient.outputs.available == 'true'
+        timeout-minutes: 5
         run: |
-          sudo apt-get update
-          sudo apt-get install -y default-jdk
+          if command -v keytool >/dev/null 2>&1; then exit 0; fi
+          sudo apt-get update -qq
+          sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            -o Acquire::Retries=3 -o Acquire::http::Timeout=20 \
+            default-jdk
 
       - name: Generate TLS certificates
         if: steps.mqclient.outputs.available == 'true'

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/.github/workflows/node.yml

@@ -19,6 +19,12 @@ on:
       - "src/**"
   workflow_dispatch:
 
+# Superseded PR runs are cancelled; push runs always finish so every pre-merge
+# commit keeps its own verdict.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
   CARGO_TERM_COLOR: always
 

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/.github/workflows/python.yml

@@ -19,6 +19,12 @@ on:
       - "src/**"
   workflow_dispatch:
 
+# Superseded PR runs are cancelled; push runs always finish so every pre-merge
+# commit keeps its own verdict.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
   CARGO_TERM_COLOR: always
 

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/.github/workflows/test-matrix.yml

@@ -6,6 +6,12 @@ on:
   pull_request:
     branches: [main, dev]
 
+# Superseded PR runs are cancelled; push runs always finish so every pre-merge
+# commit keeps its own verdict.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 env:
   CARGO_TERM_COLOR: always
 

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/CHANGELOG.md

@@ -2,6 +2,93 @@
 
 All notable changes to `mq-bridge`. Newest first.
 
+## 0.4.4
+
+### Breaking
+
+- **`idempotency` on the `file` and `object_store` sinks is replaced by `name_by`.** The old
+  field described a consequence rather than the thing it set. What it actually chooses is how
+  written objects and parts are *named*, and everything else — sortability, replay dedup,
+  whether `date_partition` applies, how errors are reported — follows from that name:
+
+  | `name_by` | Name | Consequence |
+  | --- | --- | --- |
+  | `write_time` | `<uuidv7>.<ext>`, optionally under `YYYY/MM/DD/` | unique per write, sorts by write order |
+  | `source_position` | `part-<topic>-<partition>-<start>-<end>.<ext>`, zero-padded, flat | repeats exactly on a replay, sorts by source order |
+  | `auto` (new default) | `source_position` where the input carries a replay position, else `write_time` | see below |
+
+  `idempotency: true|false` is still read and still means
+  `name_by: source_position|write_time`, so v0.4.x configs keep loading. An explicit `name_by`
+  wins over it.
+
+- **An `object_store` sink now defaults to `source_position` naming over a replayable input**
+  (Kafka, Postgres CDC, a SQL cursor, a MongoDB change stream, or a file). Key order is the
+  only order a bucket has, so this is what makes `mqb copy orders.csv s3://…` come back in
+  source order at any `concurrency` without configuring anything — and it is what removed the
+  `concurrency > 1` warning from that route. Three consequences to know about:
+  - **A re-run into the same prefix writes nothing — where the source repeats its
+    positions.** The names repeat, and a repeat is a no-op. That covers a re-read of the same
+    file, Kafka offsets, a SQL cursor and a CDC stream. A file source in `subscribe` or
+    `group_subscribe` mode is the exception: it stamps a per-run epoch, so its names stay
+    distinct across runs and a re-run writes a second copy. Where the no-op does apply it is a
+    change of behaviour — the same command used to append a second copy under fresh uuid
+    names. Use a fresh prefix, or `name_by: write_time`, if you want the old behaviour. The
+    no-op also needs the prefix listing to succeed: a replay whose batches fall on different
+    boundaries names different objects, so where the listing is denied the sink can only catch
+    a repeat that lands on the exact same name and the rest is written twice.
+  - **`date_partition` does not apply** under `source_position`; parts are written flat.
+  - **Do not mix the two families in one prefix.** uuid names and `part-…` names do not sort
+    correctly against each other. A prefix written by an older version keeps working on its
+    own; point the upgraded route at a new prefix if a reader depends on listing order.
+
+- **The `file` sink stays on `write_time` under `auto`** and must be told
+  `name_by: source_position` explicitly. There, positional naming turns `path` from a file
+  into a *directory* of part files — a change of structure, not of name, and not something to
+  derive on the operator's behalf.
+
+- `object_store` `date_partition` is now `Option<bool>` (unset = on, and only for
+  `write_time`). Side effect: the "`date_partition` is a publisher-only option" warning used
+  to fire for *every* `object_store` consumer, because the field defaulted to `true` and the
+  check could not tell that from an explicit setting. It now fires only when the field was
+  actually set.
+
+### Changed
+
+- **The `object_store` sink lists its prefix only on the `source_position` write path**, and
+  only once per publisher, immediately before its first write rather than when it opens. A
+  `write_time` sink never lists at all; a `source_position` restart into a populated prefix
+  pays for exactly one `LIST` and then skips the rest of the replay without re-encoding or
+  re-uploading. The listing cannot be deferred any further than this: `PutMode::Create` only
+  catches a repeat that lands on the same object name, so a replay whose batches fall on
+  different boundaries collides with nothing and would otherwise write every record a second
+  time under overlapping names. A listing that fails is retried on the next batch, unless it
+  failed for a reason retrying cannot clear (a denied `ListObjects`), in which case the sink
+  warns once and falls back to same-name-only deduplication.
+- **The `concurrency > 1` warning now fires only for a `write_time` name**, which is the only
+  case where write order is not source order — under `auto` over a replayable input that is
+  now nobody. It names `name_by: source_position` where the input carries a replay position
+  and `concurrency: 1` otherwise, rather than pointing at a setting that would fail on an
+  input with no position.
+- The `source_position` write path reports `SentBatch::Partial` like the ordinary path does.
+  A record that cannot be encoded splits the run instead of failing the whole batch: the
+  offsets around it are written under names covering exactly what went in, and the bad record
+  goes to the DLQ. (Encode failures are unreachable in practice today; this removes an
+  asymmetry rather than fixing an observed bug.)
+
+### Fixed
+
+- A failed prefix listing no longer fails an `object_store` batch. The error was propagated as
+  non-retryable, which dropped a replay's batch on a bucket that denies `ListObjects`, or on a
+  transient listing hiccup. It is now logged and the sink degrades to same-name-only
+  deduplication: a transient failure is retried on the next batch, while a denied `ListObjects`
+  is latched after one warning and not retried.
+
+- A route rejected for `name_by: source_position` over an input that carries no replay position
+  no longer leaves a directory behind at a `file` sink's `path`. The publisher opens before the
+  consumer, and the part-file sink creates its directory as it opens, so the rejected route used
+  to turn the operator's output *file* into an empty directory — after which a corrected run
+  failed on the leftover. The requirement is now checked before any sink opens.
+
 ## 0.4.0
 
 ### Breaking

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/Cargo.lock

@@ -3348,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.4.3"
+version = "0.4.5"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3431,7 +3431,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.4.3"
+version = "0.4.5"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3445,7 +3445,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.4.3"
+version = "0.4.5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3487,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.4.3"
+version = "0.4.5"
 dependencies = [
  "anyhow",
  "async-trait",

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.3"
+version = "0.4.5"
 
 [workspace.dependencies]
 anyhow = "1.0"

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.4.3
+Version: 0.4.5
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.5}/README.md

@@ -268,7 +268,7 @@ deduplicate on, and what a handler in the route changes about all of this, is co
 
 > **[docs/DELIVERY.md](docs/DELIVERY.md) — delivery guarantees.** Per-source identity and
 > per-sink idempotency tables, the `deduplication` middleware, Mongo `id_field` and `report_outcome`,
-> SQL `ON CONFLICT`, ClickHouse `ReplacingMergeTree`, file/object-store `idempotency: true`, what
+> SQL `ON CONFLICT`, ClickHouse `ReplacingMergeTree`, file/object-store `name_by: source_position`, what
 > handlers change, and what `mq-bridge` deliberately does not provide.
 
 The short form: point the sink at a deterministic business key (`id_field` for MongoDB, an
@@ -279,7 +279,7 @@ across every writer, so no extra state store is needed. Add the `deduplication`
 ### Cloud Object Storage (S3 / GCS / Azure)
 The `object_store` endpoint (alias `s3`) reads and writes cloud object stores — Amazon S3, Google Cloud Storage, Azure Blob, Cloudflare R2, and anything else the [`object_store`](https://crates.io/crates/object_store) crate speaks — behind the same `receive_batch` / `send_batch` API. Enable it with the `object-store` feature. Credentials and backend options are read from the process environment (`AWS_ACCESS_KEY_ID`, `AWS_ENDPOINT`, `AWS_REGION`, `GOOGLE_SERVICE_ACCOUNT`, `AZURE_STORAGE_ACCOUNT`, ...); the URL scheme picks the backend (`s3://`, `gs://`, `az://`).
 
-*   **As a sink**, each flushed batch is encoded with the same file endpoint formats (`normal` JSONL, `json`, `text`, `raw`) and written as **one immutable object** at `<prefix>/[YYYY/MM/DD/]<uuidv7>.<ext>`. Objects are write-once — nothing is appended or mutated. The uuidv7 name already sorts by write time; the optional `date_partition` prefix (on by default, derived from that same id's timestamp) is a readability / lifecycle-rule convenience. This naming applies only while `idempotency` is off — with `idempotency: true` the parts are instead named for the source range they cover and written flat under the prefix, and `date_partition` is ignored (see [Files & object storage — `idempotency`](docs/DELIVERY.md#files--object-storage--idempotency)).
+*   **As a sink**, batches are encoded with the same file endpoint formats (`normal` JSONL, `json`, `text`, `raw`) and written as whole immutable objects — write-once, nothing appended or mutated. Under `name_by: write_time` each flushed batch becomes **one object** at `<prefix>/[YYYY/MM/DD/]<uuidv7>.<ext>`; the uuidv7 name already sorts by write time, and the optional `date_partition` prefix (on by default, derived from that same id's timestamp) is a readability / lifecycle-rule convenience. Under `name_by: source_position` a batch becomes **one object per contiguous run of source positions** it covers — usually one, more when the batch spans partitions or skips offsets a replay already wrote — each named for the range it covers and written flat under the prefix, where `date_partition` does not apply. The default `name_by: auto` picks `source_position` whenever the route's input carries a replay position — Kafka, Postgres CDC, a SQL cursor, a MongoDB change stream or a file — so key order equals source order at any `concurrency` without configuring anything (see [Files & object storage — `name_by`](docs/DELIVERY.md#files--object-storage--name_by)).
 *   **As a source**, objects under the prefix are listed in key order, fetched whole, split on the delimiter, and emitted. Progress is a durable cursor holding the last fully-acked object key: set `cursor_id` and an external `checkpoint_store` (`file://`, `s3://`, `postgres://`, `mongodb://`) so a restart resumes without re-emitting. Objects are **never deleted or rewritten** — resume is non-destructive and at-least-once at object granularity (a nacked batch is redelivered; the cursor only advances once an object is fully acked). `csv` is supported on the source only.
 
 ```yaml