mq-bridge-py-basic 0.3.3__tar.gz → 0.3.5__tar.gz

{mq_bridge_py_basic-0.3.3 → mq_bridge_py_basic-0.3.5}/.github/workflows/benchmark.yml

@@ -179,6 +179,11 @@ jobs:
     name: Collect Benchmark Results
     needs: [benchmark-streams, benchmark-database, benchmark-dockerless]
     runs-on: ubuntu-latest
+    # Serialize gh-pages pushes so overlapping runs don't race the Pages build
+    # (concurrent pushes make GitHub cancel the superseded deployment -> "failed").
+    concurrency:
+      group: gh-pages-publish
+      cancel-in-progress: false
     permissions:
       contents: write
       deployments: write
@@ -248,7 +253,24 @@ jobs:
           tool: 'cargo'
           output-file-path: output.txt
           github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push on both main and dev so dev commits appear in the benchmark graph.
+          # This mints a gh-pages deployment per run; the next step prunes old ones.
           auto-push: true
           alert-threshold: '40000%'
           comment-on-alert: true
           fail-on-alert: false
+
+      # Keep only the most recent gh-pages deployments so the environment's
+      # deployment list doesn't grow unbounded (one is minted per push above).
+      - name: Prune old gh-pages deployments
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          KEEP: 20
+        run: |
+          gh api "repos/${{ github.repository }}/deployments?environment=github-pages&per_page=100" --paginate -q '.[].id' \
+            | tail -n +$((KEEP + 1)) \
+            | while read -r id; do
+                gh api -X POST "repos/${{ github.repository }}/deployments/$id/statuses" -f state=inactive >/dev/null 2>&1 \
+                  || echo "::warning::failed to mark deployment $id inactive; attempting delete anyway"
+                gh api -X DELETE "repos/${{ github.repository }}/deployments/$id" >/dev/null 2>&1 && echo "pruned $id"
+              done

{mq_bridge_py_basic-0.3.3 → mq_bridge_py_basic-0.3.5}/CONFIGURATION.md

@@ -116,6 +116,8 @@ zeromq_pipeline:
       url: "tcp://localhost:5556"
       socket_type: "push"
       bind: false
+      # format: "raw"         # send/receive raw payload bytes per frame instead of JSON-wrapping (e.g. JPEG, Protobuf)
+      # format: "raw_framed"  # like "raw" but prepends a JSON metadata frame so headers still travel. Default "json".
 
 # Route 8: PostgreSQL via SQLx
 sqlx_postgres_route:
@@ -139,6 +141,22 @@ For example, to set the Kafka topic for the `kafka_to_nats` route:
 ```sh
 export MQB__KAFKA_TO_NATS__INPUT__KAFKA__TOPIC="my-other-topic"
 ```
+#### Postgres CDC example
+
+```yaml
+orders_cdc:
+  input:
+    postgres_cdc:
+      url: "postgres://user:pass@localhost:5432/app"
+      publication: "orders_pub"   # CREATE PUBLICATION orders_pub FOR TABLE orders;
+      slot_name: "mqb_orders"       # created if missing (permanent slot, resumable)
+  output:
+    nats:
+      subject: "orders.changes"
+      url: "nats://localhost:4222"
+```
+
+Each change arrives as a `CanonicalMessage` whose payload is the flat row and whose `postgres.operation` metadata marks the operation — the same convention as MongoDB CDC, so typed handlers work identically across both. The replication transport uses the published [`pgwire-replication`](https://crates.io/crates/pgwire-replication) crate.
 
 ### NATS JetStream Notes
 

{mq_bridge_py_basic-0.3.3 → mq_bridge_py_basic-0.3.5}/Cargo.lock

@@ -218,6 +218,18 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "async-compression"
+version = "0.4.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac"
+dependencies = [
+ "compression-codecs",
+ "compression-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-executor"
 version = "1.14.0"
@@ -990,6 +1002,7 @@ dependencies = [
  "iana-time-zone",
  "js-sys",
  "num-traits",
+ "serde",
  "wasm-bindgen",
  "windows-link",
 ]
@@ -1097,6 +1110,23 @@ dependencies = [
  "tokio-util",
 ]
 
+[[package]]
+name = "compression-codecs"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
+dependencies = [
+ "compression-core",
+ "flate2",
+ "memchr",
+]
+
+[[package]]
+name = "compression-core"
+version = "0.4.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
+
 [[package]]
 name = "concurrent-queue"
 version = "2.5.0"
@@ -1250,6 +1280,16 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
 
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin 0.10.1",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -1742,9 +1782,9 @@ dependencies = [
 
 [[package]]
 name = "fast-uuid-v7"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07a7bb9c4e64a101eeec4ff43c3974178e3dd2bff832fc2c2d6edeaa14c449e5"
+checksum = "e8eaa2987e8b03da0c1659e70392d20480a595047e3db7922094ffe364e9d7b0"
 dependencies = [
  "rand 0.9.4",
 ]
@@ -1797,7 +1837,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
 dependencies = [
  "futures-core",
  "futures-sink",
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -1808,7 +1848,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be"
 dependencies = [
  "futures-core",
  "futures-sink",
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -2315,6 +2355,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+
 [[package]]
 name = "hybrid-array"
 version = "0.4.12"
@@ -2650,6 +2696,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.18"
@@ -2952,6 +3007,12 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "md5"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+
 [[package]]
 name = "memchr"
 version = "2.8.2"
@@ -3119,7 +3180,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -3130,6 +3191,7 @@ dependencies = [
  "aws-sdk-sns",
  "aws-sdk-sqs",
  "base64",
+ "byteorder",
  "bytes",
  "chrono",
  "config",
@@ -3138,6 +3200,7 @@ dependencies = [
  "flate2",
  "futures",
  "h2 0.4.14",
+ "hex",
  "http-body 1.0.1",
  "http-body-util",
  "hyper 1.10.1",
@@ -3150,7 +3213,10 @@ dependencies = [
  "mongodb",
  "mq-bridge",
  "mqi",
+ "object_store",
  "once_cell",
+ "percent-encoding",
+ "pgwire-replication",
  "prost",
  "protoc-bin-vendored",
  "rand 0.10.1",
@@ -3188,7 +3254,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3202,7 +3268,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3223,7 +3289,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.3.3"
+version = "0.3.5"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3451,6 +3517,46 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "object_store"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50"
+dependencies = [
+ "async-trait",
+ "aws-lc-rs",
+ "base64",
+ "bytes",
+ "chrono",
+ "crc-fast",
+ "form_urlencoded",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http 1.4.2",
+ "http-body-util",
+ "httparse",
+ "humantime",
+ "hyper 1.10.1",
+ "itertools 0.15.0",
+ "md-5 0.11.0",
+ "parking_lot 0.12.5",
+ "percent-encoding",
+ "quick-xml",
+ "rand 0.10.1",
+ "reqwest",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "url",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
 [[package]]
 name = "oid-registry"
 version = "0.8.1"
@@ -3689,6 +3795,28 @@ dependencies = [
  "indexmap",
 ]
 
+[[package]]
+name = "pgwire-replication"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7eec6334b2466a0163bd91c29e06a25b391c155231934aee80381b99560e240e"
+dependencies = [
+ "anyhow",
+ "base64",
+ "bytes",
+ "hmac 0.12.1",
+ "md5",
+ "rand 0.9.4",
+ "rustls 0.23.40",
+ "sha2 0.10.9",
+ "thiserror 1.0.69",
+ "tokio",
+ "tokio-rustls 0.26.4",
+ "tokio-util",
+ "tracing",
+ "webpki-roots 0.26.11",
+]
+
 [[package]]
 name = "phf"
 version = "0.13.1"
@@ -4096,6 +4224,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -4496,6 +4634,8 @@ dependencies = [
  "rustls 0.23.40",
  "rustls-pki-types",
  "rustls-platform-verifier",
+ "serde",
+ "serde_urlencoded",
  "sync_wrapper",
  "tokio",
  "tokio-rustls 0.26.4",
@@ -5027,6 +5167,18 @@ dependencies = [
  "serde_core",
 ]
 
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "serde_with"
 version = "3.21.0"
@@ -5253,6 +5405,12 @@ dependencies = [
  "lock_api",
 ]
 
+[[package]]
+name = "spin"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
+
 [[package]]
 name = "spki"
 version = "0.7.3"
@@ -5948,12 +6106,17 @@ version = "0.6.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
 dependencies = [
+ "async-compression",
  "bitflags 2.13.0",
  "bytes",
+ "futures-core",
  "futures-util",
  "http 1.4.2",
  "http-body 1.0.1",
+ "http-body-util",
  "pin-project-lite",
+ "tokio",
+ "tokio-util",
  "tower",
  "tower-layer",
  "tower-service",

{mq_bridge_py_basic-0.3.3 → mq_bridge_py_basic-0.3.5}/Cargo.toml

@@ -16,13 +16,13 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.3.3"
+version = "0.3.5"
 
 [workspace.dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
 bytes = "1"
-fast-uuid-v7 = "0.1.5"
+fast-uuid-v7 = "0.1.6"
 serde = "1.0"
 serde_json = "1.0"
 serde_yaml_ng = "0.10"
@@ -97,6 +97,10 @@ lapin = { version = "4.10", default-features = false, features = ["tokio", "rust
 # MongoDB
 mongodb = { version = "3.6", optional = true, features = ["cert-key-password"] }
 
+# Object store (checkpoint backend): cloud REST via reqwest, not aws-sdk. 0.12 is the
+# last release supporting rust-version 1.75 (0.14 needs 1.85).
+object_store = { version = "0.14", optional = true, default-features = false, features = ["aws", "gcp", "azure", "http"] }
+
 # NATS
 async-nats = { version = "0.49", optional = true }
 
@@ -134,6 +138,11 @@ aws-sdk-sns = { version = "1.93", optional = true, default-features = false, fea
 # SQLx
 sqlx = { version = "0.9", features = ["runtime-tokio", "tls-rustls", "postgres", "mysql", "sqlite", "any"], optional = true }
 
+# ClickHouse — talks the ClickHouse HTTP interface directly (JSONEachRow in/out).
+# reqwest is used rather than the typed `clickhouse` crate because that crate is
+# RowBinary/`Row`-derive only and cannot round-trip arbitrary dynamic JSON payloads.
+reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "charset", "query", "gzip"], optional = true }
+
 # ZeroMQ
 zeromq = { version = "0.6", optional = true }
 
@@ -149,6 +158,13 @@ prost = { version = "0.14", optional = true, default-features = false, features
 tokio-stream = { version = "0.1", optional = true, features = ["net"] }
 
 url = { version = "2.5", optional = true }
+percent-encoding = { version = "2.3", optional = true }
+
+# Postgres CDC (logical replication / pgoutput). pgwire-replication is a
+# narrowly-scoped wire-protocol client (COPY-both, LSN/standby, TLS/SCRAM) that
+# deliberately avoids tokio-postgres; byteorder backs the pgoutput byte decoder.
+pgwire-replication = { version = "0.3", optional = true, default-features = false, features = ["tls-rustls", "scram", "md5"] }
+byteorder = { version = "1.5", optional = true }
 
 # Generate JSON schema
 schemars = { version = "1.2", optional = true }
@@ -168,7 +184,7 @@ middleware = ["metrics", "dedup"]
 # runtime the client is loaded lazily only when an ibm-mq endpoint is opened
 # (clear error if the client isn't installed). Use `ibm-mq-static` instead for
 # static linking.
-full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "websocket", "rustls-aws-lc", "ibm-mq"]
+full = ["middleware", "mongodb", "kafka", "amqp", "nats", "mqtt", "http", "aws", "zeromq", "redis-streams", "sled", "grpc", "sqlx", "clickhouse", "websocket", "postgres-cdc", "object-store", "rustls-aws-lc", "ibm-mq"]
 
 # Functionality features
 dedup = ["sled"]
@@ -194,7 +210,7 @@ schema = ["dep:schemars"]
 kafka = ["rdkafka"]
 amqp = ["lapin", "rustls", "rustls-pemfile", "url"]
 nats = ["async-nats", "rustls", "rustls-pemfile"]
-mongodb = ["dep:mongodb"]
+mongodb = ["dep:mongodb", "url"]
 mqtt = ["rumqttc", "rustls", "tokio-rustls", "rustls-pemfile", "url"]
 http = ["dep:arc-swap", "hyper", "hyper-util", "hyper-rustls", "http-body-util", "dep:http-body", "rustls", "tokio-rustls", "rustls-pemfile", "webpki-roots", "h2", "dep:flate2", "dep:base64"]
 websocket = ["dep:tokio-websockets"]
@@ -216,6 +232,13 @@ redis-streams = ["dep:redis"]
 # Note: The grpc feature requires protobuf-src which can significantly increase build times.
 grpc = ["tonic", "prost", "tokio-stream", "tonic-prost", "tonic-prost-build", "protoc-bin-vendored", "rustls"]
 sqlx = ["dep:sqlx", "url"]
+clickhouse = ["dep:reqwest", "dep:flate2", "url"]
+# Cloud object-store checkpoint backend (S3/GCS/Azure/R2). Talks cloud REST via reqwest.
+object-store = ["dep:object_store", "url"]
+# Postgres logical-replication CDC (pgoutput). Reuses the sqlx dependency for the
+# slot-lifecycle control plane; the streaming wire protocol is pgwire-replication.
+# Needs Rust 1.88 (pgwire-replication MSRV), like the amqp feature gate.
+postgres-cdc = ["dep:pgwire-replication", "dep:byteorder", "sqlx", "dep:base64", "url", "dep:percent-encoding"]
 
 [dev-dependencies]
 config = { version = "0.15" }
@@ -223,6 +246,8 @@ serde_yaml_ng = "0.10"
 tempfile = "3.10"
 chrono = "0.4"
 metrics-util = "0.20"
+# pgoutput decoder unit tests decode hex-encoded wire fixtures.
+hex = "0.4"
 criterion = { version = "0.8", features = ["async", "async_tokio"] }
 # Used by the router_bench microbench to model the proposed lock-free route table.
 arc-swap = "1"

{mq_bridge_py_basic-0.3.3 → mq_bridge_py_basic-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.3.3
+Version: 0.3.5
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -113,9 +113,24 @@ schema = config_schema()        # the JSON Schema as a dict
 ```
 
 `config_schema()` is handy for editor validation of YAML configs too — dump it
-to a file and point your `# yaml-language-server: $schema=` line at it. The types
-are regenerated with `uv run python scripts/gen_config_types.py` (a test fails if
-they drift from the schema).
+to a file and point your `# yaml-language-server: $schema=` line at it.
+
+### Regenerating the config types
+
+`mq_bridge/config.pyi` and `mq_bridge/config.py` are **generated — do not edit by
+hand**. Regenerate them whenever the Rust config models change (e.g. adding a
+field or a new endpoint). Because the generator reads the schema from the
+compiled extension, you must rebuild first:
+
+```bash
+# from python/mq-bridge-py/
+uv run maturin develop                              # rebuild the extension
+uv run --no-sync python scripts/gen_config_types.py # regenerate config.pyi / config.py
+```
+
+Then commit the updated `config.pyi`/`config.py`. `tests/test_config_types.py`
+asserts the checked-in output matches the schema, so CI (the "Python package
+smoke test" job) fails if you skip this after a models change.
 
 ## Running a route