mq-bridge-basic 0.4.19__tar.gz → 0.4.20__tar.gz

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/.dockerignore

@@ -5,7 +5,7 @@ logs
 target
 **/target
 **/node_modules
-apps/mq-bridge-app/dev/docs/book
+docs/book/book
 
 # Local build output
 **/.venv

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/.github/dependabot.yml

@@ -9,7 +9,8 @@ updates:
     directory: "/" # Location of package manifests
     target-branch: "dev"
     schedule:
-      interval: "monthly"
+      # Same cadence as the app workspace below, so the two lockfiles do not drift.
+      interval: "weekly"
     groups:
       minor-and-patch:
         applies-to: version-updates

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/.github/workflows/app.yml

@@ -51,6 +51,7 @@ jobs:
           cache-workspace-crates: true
 
       - name: Install system dependencies
+        timeout-minutes: 5
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev pkg-config zlib1g-dev protobuf-compiler libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
 
       - name: Test desktop crate
@@ -116,6 +117,7 @@ jobs:
           cache-dependency-path: apps/mq-bridge-app/package-lock.json
 
       - name: Install system dependencies
+        timeout-minutes: 5
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev pkg-config zlib1g-dev protobuf-compiler
 
       - name: Install frontend dependencies
@@ -193,6 +195,7 @@ jobs:
           cache-workspace-crates: true
 
       - name: Install system dependencies
+        timeout-minutes: 5
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev pkg-config zlib1g-dev protobuf-compiler
 
       - name: Run integration suite
@@ -224,6 +227,7 @@ jobs:
           cache-workspace-crates: true
 
       - name: Install system dependencies
+        timeout-minutes: 5
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev pkg-config zlib1g-dev protobuf-compiler
 
       # The test drives the CLI through `env!("CARGO_BIN_EXE_mq-bridge-app")`, an
@@ -294,6 +298,7 @@ jobs:
           chmod +x "$cli" docker-suite/test-binary
 
       - name: Install runtime libraries
+        timeout-minutes: 5
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev zlib1g-dev
 
       # The engine's own integration stacks, so there is one definition per

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/.github/workflows/ci.yml

@@ -120,6 +120,11 @@ jobs:
       - name: C plugin header is current and a C plugin loads
         run: cargo test -p mq-bridge-plugin-fixture --test plugin_c_header --test plugin_c_abi
 
+      - name: C library header is current and a C program links and runs
+        run: cargo test -p mq-bridge-c --no-default-features
+        env:
+          MQB_C_ASAN: "1" # AddressSanitizer; on Linux it also fails on leaks
+
       - name: Memory leak soak (route commit-task JoinSet)
         run: cargo test --test memory_leak_test --features=full -- --ignored --nocapture
 

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/.github/workflows/docs.yml

@@ -5,7 +5,7 @@ on:
     branches: [main]
     paths:
       - "docs/**"
-      - "apps/mq-bridge-app/dev/docs/**"
+      - "llms.txt"
       - "apps/mq-bridge-app/crates/cli/examples/gen_url_docs.rs"
       - ".github/workflows/docs.yml"
   # The benchmark workflow pushes new data to the gh-pages branch; republish the
@@ -38,14 +38,18 @@ jobs:
           mdbook-version: "0.5.4"
 
       - name: Vendor engine docs
-        run: bash apps/mq-bridge-app/dev/docs/sync-engine-docs.sh
+        run: bash docs/book/sync-engine-docs.sh
 
       - name: Build book
-        run: mdbook build apps/mq-bridge-app/dev/docs
+        run: mdbook build docs/book
 
-      # mdbook copies non-Markdown files from the book source; fail if that ever stops.
-      - name: Check llms.txt is published
-        run: test -s apps/mq-bridge-app/dev/docs/book/llms.txt
+      # llms.txt lives at the repository root; publish it with the book.
+      - name: Publish llms.txt
+        run: cp llms.txt docs/book/book/llms.txt
+
+      # Canonical URLs, meta descriptions, sitemap.xml and llms-full.txt.
+      - name: Postprocess book
+        run: python3 docs/book/postprocess.py
 
       # The benchmark dashboard lives on the gh-pages branch. Copy it into the
       # book so this workflow is the only thing that publishes to Pages.
@@ -61,8 +65,8 @@ jobs:
         run: |
           src=.benchmark-data/dev/bench
           if [ -d "$src" ]; then
-            mkdir -p apps/mq-bridge-app/dev/docs/book/dev
-            cp -R "$src" apps/mq-bridge-app/dev/docs/book/dev/bench
+            mkdir -p docs/book/book/dev
+            cp -R "$src" docs/book/book/dev/bench
           else
             echo "::warning::no benchmark data on gh-pages; /dev/bench/ will be missing"
           fi
@@ -70,7 +74,7 @@ jobs:
       - name: Upload Pages artifact
         uses: actions/upload-pages-artifact@v5
         with:
-          path: apps/mq-bridge-app/dev/docs/book
+          path: docs/book/book
 
   deploy:
     name: Deploy to Pages

mq_bridge_basic-0.4.20/.github/workflows/examples.yml

@@ -0,0 +1,62 @@
+name: Examples
+
+# Runs the use-case examples under examples/ the way their READMEs describe.
+on:
+  push:
+    branches: [main]
+    paths:
+      - "examples/postgres-cdc-to-qdrant/**"
+      - "examples/transactional-outbox/**"
+      - "examples/test-without-a-broker/**"
+      - ".github/workflows/examples.yml"
+  pull_request:
+    paths:
+      - "examples/postgres-cdc-to-qdrant/**"
+      - "examples/transactional-outbox/**"
+      - "examples/test-without-a-broker/**"
+      - ".github/workflows/examples.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: examples-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker:
+    name: ${{ matrix.example }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        example: [postgres-cdc-to-qdrant, transactional-outbox]
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Run the example's smoke test
+        working-directory: examples/${{ matrix.example }}
+        run: ./test.sh
+
+  test-without-a-broker:
+    name: test-without-a-broker
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - name: Rust
+        working-directory: examples/test-without-a-broker
+        run: cargo test --locked
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Python
+        working-directory: examples/test-without-a-broker/python
+        run: |
+          pip install -r requirements.txt
+          pytest -q

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/AGENTS.md

@@ -186,8 +186,8 @@ The grammar and examples are in [CONFIGURATION.md](docs/CONFIGURATION.md).
 2. **New middleware**: Add to `middleware/mod.rs`, implement middleware wrapper types
 3. **Configuration changes**: Update `models.rs` with new config structs, add to `EndpointType`/`Middleware` enums
 4. **Tests**: Add integration tests in `tests/integration/`, update Docker Compose if needed
-5. **Documentation**: Update README.md and add doc comments; when middleware or endpoint behaviour changes, also check the book's hand-written pages in `apps/mq-bridge-app/dev/docs/cookbook` and `operations`
+5. **Documentation**: Update README.md and add doc comments; when middleware or endpoint behaviour changes, also check the book's hand-written pages in `docs/book/cookbook` and `operations`
 
 ### Dependencies
 
-Dependencies and the feature that gates each optional one are in `Cargo.toml`, with comments explaining the non-obvious choices. The crate has no default features.
+Dependencies and the feature that gates each optional one are in `Cargo.toml`, with comments explaining the non-obvious choices. The only default features are `file` and `dir-spool`; an endpoint whose feature is off is served by a registered factory of the same name if there is one (see `plugins/full`).

{mq_bridge_basic-0.4.19 → mq_bridge_basic-0.4.20}/CHANGELOG.md

@@ -2,6 +2,154 @@
 
 All notable changes to `mq-bridge`. Newest first.
 
+## 0.4.20
+
+Kafka outputs publish faster with their default settings. If you depend on `mq-bridge` with
+`default-features = false` and use the `file` or `dir_spool` endpoint, read "Behaviour
+changes" first: both are Cargo features now.
+
+### Behaviour changes
+
+- **`mqb --ui` listens on `127.0.0.1:9091`, not on `0.0.0.0:9091`.** The UI has no login,
+  and the old default put it on the network. The Docker image keeps `0.0.0.0:9091` through
+  `ENV MQB_UI_DEFAULT_ADDR`. To reach the UI from another machine, set `ui_addr` in the
+  config; a start on such an address logs a warning.
+- **A web UI on a loopback address refuses a request with a foreign `Host` header** (403).
+  This stops a web page from reaching the UI through DNS rebinding. Behind a reverse proxy
+  or a port forward that sends another host name, list the names in `MQB_UI_ALLOWED_HOSTS`.
+- **`GET /config` of the web UI no longer returns secrets** in any storage mode except
+  `unencrypted` and `temporary_messages`. Each secret is shown as a reference to the key it
+  is stored under, such as `${MQB__PUBLISHERS__<id>__HTTP__BASIC_AUTH__1}`; a URL keeps
+  everything but its password. A save replaces each reference by the stored value, so
+  cloning, copying and renaming an endpoint keep its secrets. A reference to a key that is
+  not stored is refused with 400. A config exported from the UI holds references, and a
+  script that read secrets from this endpoint no longer gets them.
+- **An HTTP output with `tls.required: true` and an `http://` URL fails at start.** It sent
+  its requests unencrypted before. An `http://` output to another host that carries
+  `basic_auth` or custom headers logs a warning.
+
+- **`file` and `dir_spool` are Cargo features: `file` and `dir-spool`.** Both endpoints
+  were compiled into every build. They are now in the default features, and in `full` and
+  `portable`; `object-store` enables `file`.
+  - A dependency with default features, `full` or `portable` builds what it built before.
+  - A dependency with `default-features = false` loses both endpoints. A route that names
+    one fails at start with `Unsupported consumer endpoint type` (or `publisher`), and code
+    that names `endpoints::file` or `endpoints::dir_spool` no longer compiles. To keep them:
+
+    ```toml
+    mq-bridge = { version = "0.4", default-features = false, features = ["file", "dir-spool"] }
+    ```
+
+  - The config types (`FileConfig`, `DirSpoolConfig`) and the `EndpointType` variants stay
+    in every build, so a configuration still parses.
+  - `mq-bridge-app`, the Python package and the Node.js package are unchanged; they enable
+    both.
+
+### Added
+
+- **A C library.** The new `c/mq-bridge-c` crate builds `libmq_bridge` (shared and static)
+  with the header `include/mq_bridge.h`: messages, publishers (send, batch, request/reply),
+  pull consumers with commit and per-batch ack/nack, and routes with handlers, configured
+  with the same YAML or JSON as the other bindings. Every call blocks. The header is
+  generated and carries no ABI stability promise: it ships with its library, and
+  `mqb_api_version()` detects a mismatch. There are no prebuilt archives yet; see
+  `c/mq-bridge-c/README.md`.
+- **Plugin tables registered from the host program.** `plugin::register_plugin_table`
+  (C: `mqb_register_plugin`) registers an `MqbPluginVTable` that lives in the running
+  program, so a C or C++ application defines custom endpoints and middleware without
+  building a separate plugin library.
+- **Live status in the CLI, the UI and MCP.** `mqb status` lists every route and consumer that
+  any `mqb` process of your user is running on the machine, with state, current and average
+  rate, total, pending and uptime. On a terminal it redraws every second; `--no-watch` prints
+  once and `--json` prints the records. The
+  UI has a **Status** tab with the same rows and the last 60 seconds of each rate, and the MCP
+  server has a read-only `bridge_status` tool. `mqb copy` and headless config runs now report
+  rates and totals too. Error text stays out of these records unless
+  `MQB_STATUS_SHOW_ERRORS=1` is set for both the reporting and the reading process.
+- **`http_bulk` reads an open stream.** `read.stream: sse` or `ndjson` sends one request and
+  turns every server-sent event or line of the response into a message, instead of asking
+  for pages. An SSE source resumes with `Last-Event-ID` from the last acknowledged event, an
+  NDJSON source from a field named by `cursor.item`; both save the position in
+  `checkpoint_store` and connect again when the response ends. With a stream,
+  `request_timeout_ms` is the longest silence, not the length of the response. This is the
+  way to consume a remote SSE feed or a one-response export; the `http` endpoint's
+  `receive_streamable` and `stream_response_to` are unchanged.
+- **Named `http_bulk` endpoints from your own crate.** `typesense` and `elasticsearch` are
+  `http_bulk` configurations behind a name, and that table was closed.
+  `endpoints::http_bulk::register_preset_with(name, schema, resolve)` registers one of your
+  own as a `custom` endpoint; `preset_schema`, `preset_base_url` and `preset_segment` are the
+  helpers the built-in ones use.
+- **Every built-in endpoint as one plugin library.** The new `plugins/full` crate builds
+  `libmq_bridge_full`, which registers a factory for each built-in endpoint under its own
+  name. A host compiled without an endpoint's feature and with this library loaded keeps
+  using `kafka:`, `file:` and the rest unchanged: a built-in the build has no feature for
+  is now served by a factory registered under the same name, and fails as before when there
+  is none. The library has its own engine, so `memory` channels, `ref` names and factories
+  registered in the host are not shared with the endpoints it creates.
+- **`plugin::disable_plugin_loading()`.** After this call the process opens no plugin
+  library any more, neither by path nor by discovery, and there is no way to switch it back
+  on. `MQB_PLUGIN_DISCOVERY=0` only covers discovery and has to be set in the environment;
+  this is for a host that must rule out native code loaded at runtime.
+- **HTTP input: `header_read_timeout_ms` and `max_body_bytes`.** The first closes an HTTP/1
+  connection whose request head is not complete in time, the second sets the largest request
+  body, also after decompression (default 256 MiB, as before). The timeout is off by default:
+  with `30000` set, a debug build served about 15 to 20 % fewer requests per second on a
+  `http` to `response` route. Set it on a listener that is reachable from an untrusted network.
+
+### Changed
+
+- **CSV files read at their 0.4.15 speed again.** The quote scan that decides where a CSV
+  record ends had become about 15% slower in 0.4.16, when it learned to handle a record
+  read in pieces. It now jumps from quote to quote instead of visiting every byte, with the
+  same result. A 1,000,000-row CSV → JSONL copy on an Apple M1 takes 0.35 s, against 0.41 s
+  with 0.4.19.
+- **CSV files and objects are written faster.** A row whose keys follow the header's
+  columns, as every row after the first usually does, is written without building a key
+  map first, nested objects under `nested: flatten` included. A 1,000,000-row JSONL → CSV
+  copy on an Apple M1 takes 1.12 s instead of 1.45 s, and 0.96 s instead of 2.03 s when
+  each row flattens a nested object. The output is byte-identical.
+- **`mqb copy --drain` exits within 5 ms of the last row.** It noticed the end of a route by
+  polling every 50 ms, which added up to 50 ms to every copy, noticeable on short ones.
+- **Kafka publishing no longer waits 5 ms per batch.** A Kafka output awaits the delivery of
+  every batch, and the producer's `linger.ms` of 5 held each batch back for that long before
+  sending it. The default is now 1 ms, which still sends a batch as one request per partition.
+  With the defaults (`batch_size: 512`, `concurrency: 1`) a local broker took about 71,000
+  messages per second instead of 29,000. `delayed_ack: true`, which awaits nothing, keeps
+  5 ms; the producer a Kafka input uses for replies also moves to 1 ms. A
+  `linger.ms` in `producer_options` overrides the default as before.
+
+### Fixed
+
+- `mq-bridge-app`: saving a config in `sensitive` or `durable` mode removes the secrets
+  that an earlier `balanced` save left in the `.env` file.
+- Desktop app: the list of stored secrets keeps the keys of earlier saves, so "delete
+  secrets" also reaches the secret of a renamed or removed endpoint.
+- **Passwords no longer appear in connect logs and error text.** The AMQP and MQTT connect
+  lines at INFO printed the broker URL with its password. Errors from the HTTP and WebSocket
+  outputs quoted it too. All now show `user:***@host`; the HTTP output's `status()` target
+  does the same.
+- **`mqb copy` hides more credentials.** The `copy route started` line now also blanks
+  `basic_auth`, `custom_headers`, `headers`, API keys and the credentials of a nested endpoint
+  URI (`fanout:?to=…`), percent-encoded or not. Errors about a URI or a middleware spec quote
+  it with the same redaction instead of as typed.
+- **`cookie_jar` stores every cookie of a response.** A response with several `Set-Cookie`
+  headers left only the last one in the jar. The HTTP output now keeps them all in the
+  `set-cookie` metadata, one per line, and an HTTP input that replies with that metadata
+  sends one header per line.
+- **HTTP `stream_response_to`: a response that breaks before its first item no longer writes
+  to the stream.** The send already failed as retryable, but an error and an end marker went to
+  the stream first, so its reader saw a closed stream and a retry refilled it behind the
+  reader's back. Now nothing is written until the first item. A response that breaks later is
+  unchanged: it ends with an item marked `http_stream_error` and the send is acknowledged.
+- **`postgres_cdc`: a `truncate` event no longer carries `postgres.key`.** It had `[null]`, one
+  `null` per key column, which a sink that deletes or upserts by key would take for a row. The
+  event keeps a deterministic `message_id`, also on tables without a primary key; that id
+  differs from the one earlier versions produced.
+- **A large batch no longer fails a SQL output with an `insert_query`.** A batch that needed
+  more bind parameters than the database takes in one statement (65,535 on Postgres) failed
+  with `too many arguments for query`. It is now written as several statements in one
+  transaction, as `columns: auto` already did.
+
 ## 0.4.19
 
 CSV in more shapes: the reader takes other separators and quoting, and a CSV sink no longer
@@ -368,7 +516,7 @@ Each of these changes what an existing setup sees. Check them before upgrading.
   SOURCE, TARGET and `--filter` of a `copy --resume` job. `reset` prints the old value so
   `set --value` can undo it. Works for SQL, ClickHouse and object-store cursor readers and
   MongoDB change streams; Postgres CDC is refused, its slot is authoritative. See
-  [Checkpoints](apps/mq-bridge-app/dev/docs/cookbook/checkpoints.md#inspecting-and-editing-a-checkpoint).
+  [Checkpoints](docs/book/cookbook/checkpoints.md#inspecting-and-editing-a-checkpoint).
 - **`CheckpointStore::clear`**, implemented for the file, SQL, MongoDB and object-store
   backends. The default returns an error, so existing implementations still compile.