mq-bridge-py-basic 0.2.18__tar.gz → 0.2.20__tar.gz

{mq_bridge_py_basic-0.2.18 → mq_bridge_py_basic-0.2.20}/.github/workflows/release.yml

@@ -50,13 +50,24 @@ jobs:
     runs-on: ubuntu-latest
     needs: release
     environment: release
+    permissions:
+      # Required for crates.io Trusted Publishing (OIDC).
+      contents: read
+      id-token: write
     steps:
-      - uses: actions/checkout@v4
-      
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
-      
+
+      # Exchange the GitHub OIDC token for a short-lived crates.io token.
+      - name: Authenticate to crates.io (Trusted Publishing)
+        uses: rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18 # v1.0.5
+        id: auth
+
       - name: Publish to crates.io
         run: cargo publish
         env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}

{mq_bridge_py_basic-0.2.18 → mq_bridge_py_basic-0.2.20}/Cargo.lock

@@ -299,7 +299,7 @@ dependencies = [
  "tokio-rustls 0.26.4",
  "tokio-stream",
  "tokio-util",
- "tokio-websockets",
+ "tokio-websockets 0.10.1",
  "tracing",
  "tryhard",
  "url",
@@ -3057,9 +3057,10 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.2.18"
+version = "0.2.20"
 dependencies = [
  "anyhow",
+ "arc-swap",
  "async-channel",
  "async-nats",
  "async-trait",
@@ -3108,7 +3109,7 @@ dependencies = [
  "tokio",
  "tokio-rustls 0.26.4",
  "tokio-stream",
- "tokio-tungstenite",
+ "tokio-websockets 0.13.2",
  "tonic",
  "tonic-prost",
  "tonic-prost-build",
@@ -3123,7 +3124,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.2.18"
+version = "0.2.20"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3131,6 +3132,7 @@ dependencies = [
  "fast-uuid-v7",
  "mq-bridge",
  "pyo3",
+ "schemars",
  "serde",
  "serde_json",
  "serde_yaml_ng",
@@ -4883,6 +4885,12 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "sha1_smol"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
+
 [[package]]
 name = "sha2"
 version = "0.10.9"
@@ -5549,18 +5557,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-tungstenite"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c"
-dependencies = [
- "futures-util",
- "log",
- "tokio",
- "tungstenite",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.7.18"
@@ -5597,6 +5593,25 @@ dependencies = [
  "webpki-roots 0.26.11",
 ]
 
+[[package]]
+name = "tokio-websockets"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dad543404f98bfc969aeb71994105c592acfc6c43323fddcd016bb208d1c65cb"
+dependencies = [
+ "base64",
+ "bytes",
+ "fastrand",
+ "futures-core",
+ "futures-sink",
+ "http 1.4.2",
+ "httparse",
+ "sha1_smol",
+ "simdutf8",
+ "tokio",
+ "tokio-util",
+]
+
 [[package]]
 name = "toml"
 version = "1.1.2+spec-1.1.0"
@@ -5862,22 +5877,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tungstenite"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8"
-dependencies = [
- "bytes",
- "data-encoding",
- "http 1.4.2",
- "httparse",
- "log",
- "rand 0.9.4",
- "sha1 0.10.6",
- "thiserror 2.0.18",
-]
-
 [[package]]
 name = "typed-builder"
 version = "0.22.0"

{mq_bridge_py_basic-0.2.18 → mq_bridge_py_basic-0.2.20}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.2.18"
+version = "0.2.20"
 
 [workspace.dependencies]
 anyhow = "1.0"
@@ -26,7 +26,7 @@ fast-uuid-v7 = "0.1.5"
 serde = "1.0"
 serde_json = "1.0"
 serde_yaml_ng = "0.10"
-tokio = "1"
+tokio = "1.34"  # `Receiver::recv_many` (HTTP source batch drain) needs 1.34+.
 tracing = "0.1"
 
 [profile.release]
@@ -46,6 +46,7 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "ti
 # Note on MSRV for optional features:
 # The base crate keeps MSRV 1.75 with default = [].
 # - "amqp" (lapin 4.10) requires Rust 1.88+; "http" (hyper-rustls 0.27) requires Rust 1.85+.
+# - "websocket" (tokio-websockets 0.13) requires Rust 1.89+.
 # - "aws" (aws-sdk-*) may require a newer stable toolchain than the base MSRV.
 # Use `rustup override set stable` or a `rust-toolchain.toml` file when enabling these features.
 async-trait.workspace = true
@@ -109,6 +110,7 @@ rumqttc = { version = "0.25", default-features = false, features = ["use-rustls-
 mqi = { version = "0.3", optional = true }
 
 # HTTP
+arc-swap = { version = "1", optional = true }
 hyper = { version = "1.8", optional = true, features = ["client", "server", "http1", "http2"] }
 hyper-util = { version = "0.1", optional = true, features = ["tokio", "client", "client-legacy", "server", "http1", "http2"] }
 hyper-rustls = { version = "0.27", optional = true, default-features = false, features = ["http1", "http2", "tls12", "logging", "native-tokio"] }
@@ -117,7 +119,7 @@ http-body = { version = "1.0", optional = true }
 h2 = { version = "0.4", optional = true }
 flate2 = { version = "1.0", optional = true }
 base64 = { version = "0.22", optional = true }
-tokio-tungstenite = { version = "0.29", optional = true }
+tokio-websockets = { version = "0.13", optional = true, default-features = false, features = ["client", "server", "sha1_smol", "fastrand"] }
 
 # AWS
 aws-config = { version = "1.8", optional = true, default-features = false, features = ["behavior-version-latest", "rt-tokio", "rustls"] }
@@ -178,8 +180,8 @@ amqp = ["lapin", "rustls", "rustls-pemfile", "url"]
 nats = ["async-nats", "rustls", "rustls-pemfile"]
 mongodb = ["dep:mongodb"]
 mqtt = ["rumqttc", "rustls", "tokio-rustls", "rustls-pemfile", "url"]
-http = ["hyper", "hyper-util", "hyper-rustls", "http-body-util", "dep:http-body", "rustls", "tokio-rustls", "rustls-pemfile", "webpki-roots", "h2", "dep:flate2", "dep:base64"]
-websocket = ["dep:tokio-tungstenite"]
+http = ["dep:arc-swap", "hyper", "hyper-util", "hyper-rustls", "http-body-util", "dep:http-body", "rustls", "tokio-rustls", "rustls-pemfile", "webpki-roots", "h2", "dep:flate2", "dep:base64"]
+websocket = ["dep:tokio-websockets"]
 aws = ["aws-config", "aws-sdk-sqs", "aws-sdk-sns", "dep:base64"]
 ibm-mq = ["mqi"]
 zeromq = ["dep:zeromq"]
@@ -194,6 +196,8 @@ tempfile = "3.10"
 chrono = "0.4"
 metrics-util = "0.20"
 criterion = { version = "0.8", features = ["async", "async_tokio"] }
+# Used by the router_bench microbench to model the proposed lock-free route table.
+arc-swap = "1"
 mq-bridge = { path = ".", features = ["test-utils", "rustls-ring"] }
 reqwest = { version = "0.13", features = ["stream"] }
 
@@ -207,6 +211,12 @@ name = "performance_bench"
 harness = false
 required-features = ["test-utils"]
 
+# Isolates the per-request HTTP route lookup (Mutex<HashMap> vs ArcSwap).
+# No external services or features required.
+[[bench]]
+name = "router_bench"
+harness = false
+
 [[test]]
 name = "integration_test"
 required-features = ["test-utils"]

mq_bridge_py_basic-0.2.20/PKG-INFO

@@ -0,0 +1,191 @@
+Metadata-Version: 2.4
+Name: mq-bridge-py-basic
+Version: 0.2.20
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Requires-Dist: typing-extensions>=4.0
+License-File: LICENSE.python
+Summary: Python bindings for mq-bridge (basic: all-platform messaging set)
+Keywords: messaging,tokio,pyo3,maturin,mq-bridge
+License: MIT
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/marcomq/mq-bridge
+Project-URL: Issues, https://github.com/marcomq/mq-bridge/issues
+Project-URL: Repository, https://github.com/marcomq/mq-bridge
+
+# mq-bridge Python bindings
+
+Thin Python bindings for the Rust `mq-bridge` core.
+
+## Install
+
+Pick exactly one distribution. Both install the same import path: `mq_bridge`.
+
+| Package | Install | Includes |
+| :--- | :--- | :--- |
+| Full | `pip install mq-bridge-py` | Basic set plus Kafka, AWS, gRPC, MongoDB, SQLx |
+| Basic | `pip install mq-bridge-py-basic` | HTTP, NATS, MQTT, AMQP, WebSocket, ZeroMQ, middleware |
+
+Memory and file endpoints are always present in both packages. Use `mq-bridge-py-basic` when you want the lean all-platform wheel set. Use `mq-bridge-py` when you need Kafka or the heavier non-messaging backends.
+
+The public API stays close to mq-bridge itself:
+
+- `Route.from_yaml(path, name)` loads one named route from a YAML file
+- `Route.from_yaml_str(text, name)` / `Route.from_config(mapping, name)` build a route from an in-memory YAML string or a Python `dict`, no file required
+- `Route.with_handler(...)` attaches a raw `Message` handler, with lazy `json()`/`text()` readers and `with_json()`/`with_payload()` response helpers
+- `Route.add_handler(kind, ...)` uses mq-bridge's `kind` dispatch and delivers decoded JSON
+- `RetryableError` and `NonRetryableError` let Python handlers signal retry intent
+- `Publisher.from_yaml(path, name)` (plus `from_yaml_str` / `from_config`) loads one named publisher
+- `Publisher.send_json(...)` and `Publisher.request_json(...)` serialize Python JSON values in Rust
+
+The Python surface is synchronous and blocking. Tokio, broker I/O, routing, and batching all stay in Rust.
+
+## Config types and schema
+
+`mq-bridge-app` can create and test route and endpoint JSON/YAML through its UI.
+It does not replace your Python code or handlers, but it is useful when you want
+a known-good connection and route shape before pasting the configuration into
+Python. Load the generated config with `Route.from_config`, `Route.from_yaml`,
+`Publisher.from_config`, or `Publisher.from_yaml`.
+
+For the `from_config` / `from_yaml_str` mappings, `mq_bridge.config` ships
+`TypedDict` definitions so editors autocomplete the config keys (`input`,
+`output`, `batch_size`, every transport config, middleware, …):
+
+```python
+from mq_bridge import Route
+from mq_bridge.config import ConfigDocument
+
+config: ConfigDocument = {
+    "routes": {
+        "orders": {
+            "input": {"memory": {"topic": "orders.in", "capacity": 1600}},
+            "output": {"response": {}},
+            "batch_size": 128,
+        }
+    }
+}
+route = Route.from_config(config, "orders")
+```
+
+These types are generated from the JSON Schema, which the extension produces on
+demand from the Rust models — there is no checked-in schema copy to drift:
+
+```python
+from mq_bridge import config_schema
+
+schema = config_schema()        # the JSON Schema as a dict
+```
+
+`config_schema()` is handy for editor validation of YAML configs too — dump it
+to a file and point your `# yaml-language-server: $schema=` line at it. The types
+are regenerated with `uv run python scripts/gen_config_types.py` (a test fails if
+they drift from the schema).
+
+## Running a route
+
+`Route.run()` **blocks the calling thread** until another thread calls `stop()` —
+it deploys the route and then parks. This is convenient for a process whose only
+job is the route, but it is a common trap: nothing after `route.run()` executes
+until the route stops.
+
+To keep running Python code after the route is up, use `start()` (non-blocking)
+or the context-manager form:
+
+```python
+route = Route.from_config(config, "orders_route").with_handler(handle)
+
+# Non-blocking: deploys, returns, and runs on a background thread.
+route.start()
+publisher.send_json({"order_id": 42}, {"kind": "order.created"})
+route.stop()
+route.join()   # optional: wait for a clean shutdown
+
+# Or scope it to a block — starts on enter, stops + joins on exit:
+with Route.from_config(config, "orders_route").with_handler(handle):
+    publisher.send_json({"order_id": 42}, {"kind": "order.created"})
+```
+
+Configuration/connection errors surface from `start()` itself, not from a
+background thread. `run()` remains available for the blocking single-route case.
+
+## Tuning (environment variables)
+
+These knobs are read from the environment at startup:
+
+| Variable | Default | Effect |
+| :--- | :--- | :--- |
+| `MQ_BRIDGE_PY_HANDLER_EXECUTOR` | `worker` | `worker` runs handlers on a dedicated interpreter thread that coalesces queued batches under one GIL acquisition (best under load); `direct` calls the handler inline. |
+| `MQ_BRIDGE_PY_HANDLER_CONCURRENCY` | CPU count | Max in-flight handler batches. `0` disables the limit. |
+| `MQ_BRIDGE_PY_GC_MODE` | `default` | `default` leaves CPython's cyclic GC alone; `count` disables it and runs `gc.collect()` every N messages; `off` disables it entirely (pure refcounting). |
+| `MQ_BRIDGE_PY_GC_THRESHOLD` | `100000` | Messages between collections when `MQ_BRIDGE_PY_GC_MODE=count`. |
+
+## Local development
+
+`uv` is a good fit here for the Python-side developer workflow, while `maturin` stays the build backend:
+
+```bash
+cd python/mq-bridge-py
+uv sync --group dev --no-install-project
+uv run maturin develop
+uv run pytest -q
+```
+
+Performance smoke tests are skipped by default because they start routes and measure
+local throughput:
+
+```bash
+cd python/mq-bridge-py
+MQ_BRIDGE_RUN_PERF_TESTS=1 uv run pytest -q -m performance
+```
+
+## Examples
+
+Raw message handler:
+
+```bash
+cd python/mq-bridge-py
+uv run python examples/raw_route.py
+```
+
+Kind-based JSON handler:
+
+```bash
+cd python/mq-bridge-py
+uv run python examples/json_route.py
+```
+
+Memory benchmark:
+
+```bash
+cd python/mq-bridge-py
+uv run maturin develop --release
+uv run python examples/bench_memory.py --messages 100000
+```
+
+## Analysis
+
+HTTP comparison benchmark, driven by a native load generator (`wrk`) so the
+client is never the bottleneck. It boots each server itself (mq-bridge in worker
+and direct executor modes, plus FastAPI, Starlette, Sanic, aiohttp, and
+FastStream when installed) and drives each with `wrk`:
+
+```bash
+cd python/mq-bridge-py
+uv run maturin develop --release
+uv sync --group bench   # optional Python HTTP peers
+uv run python analysis/bench_http_native.py --connections 1,8,32 --duration 8
+```
+
+Requires `wrk` on `PATH` (`brew install wrk`). The FastStream target compares its
+ASGI custom-route path over Uvicorn; it is not a broker-backed subscriber/publisher
+benchmark. The examples use included sample configs or create temporary configs.
+

{mq_bridge_py_basic-0.2.18 → mq_bridge_py_basic-0.2.20}/README.md

@@ -157,6 +157,12 @@ The `response` output endpoint sends a reply back to the original requester. Thi
 There is a separate repository for running mq-bridge as a standalone app, for example as a Docker container configured via YAML or environment variables:
 https://github.com/marcomq/mq-bridge-app
 
+### Configuration-first workflow
+
+`mq-bridge-app` can be used to create and test route and endpoint configurations through a UI. The generated JSON/YAML can then be copied into an application and loaded by the Rust library or the Python bindings.
+
+This does not replace application code or handlers, but it is useful when you want a known-good connection and route shape before pasting the configuration into code. For Python projects, routes and publishers are commonly loaded from JSON/YAML with `Route.from_config`, `Route.from_yaml`, `Publisher.from_config`, or `Publisher.from_yaml`.
+
 ### Programmatic Handlers
 
 For business logic, `mq-bridge` provides a handler layer separate from transport-level middleware. This is where message-specific code usually belongs.

{mq_bridge_py_basic-0.2.18 → mq_bridge_py_basic-0.2.20}/benches/performance_bench.rs

@@ -648,7 +648,7 @@ pub mod websocket_helper {
         let websocket_config = WebSocketConfig {
             url: "127.0.0.1:0".to_string(),
             path: Some("/bench".to_string()),
-            internal_buffer_size: Some(super::PERF_TEST_MESSAGE_COUNT * 2),
+            routed_queue_capacity: Some(super::PERF_TEST_MESSAGE_COUNT * 2),
             ..Default::default()
         };