mq-bridge-py-basic 0.4.0__tar.gz → 0.4.1__tar.gz

{mq_bridge_py_basic-0.4.0 → mq_bridge_py_basic-0.4.1}/Cargo.lock

@@ -3348,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3431,7 +3431,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3445,7 +3445,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3487,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.4.0"
+version = "0.4.1"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3529,13 +3529,14 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
 
 [[package]]
 name = "napi"
-version = "3.12.0"
+version = "3.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f71d6bc097c4a6eb853c3f24991ab8c9f50f57d1f719e305175541482217e36"
+checksum = "459197f1592f4c3dbbf9c1b13f5a4599a343e4ef66b96bc340e2a518b36a6662"
 dependencies = [
  "bitflags 2.13.1",
  "ctor",
  "futures",
+ "libc",
  "napi-build",
  "napi-sys",
  "nohash-hasher",
@@ -3547,15 +3548,15 @@ dependencies = [
 
 [[package]]
 name = "napi-build"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
+checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
 
 [[package]]
 name = "napi-derive"
-version = "3.6.2"
+version = "3.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d9002b2940f0184444754546e0fcd15182f56948e6f381968b019d549387c42"
+checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
 dependencies = [
  "convert_case 0.11.0",
  "ctor",
@@ -3567,9 +3568,9 @@ dependencies = [
 
 [[package]]
 name = "napi-derive-backend"
-version = "6.1.1"
+version = "6.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
+checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
 dependencies = [
  "convert_case 0.11.0",
  "proc-macro2",
@@ -4791,9 +4792,9 @@ dependencies = [
 
 [[package]]
 name = "rcgen"
-version = "0.14.8"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
+checksum = "091e7a8e7d86e6feb87a27ce8e2cba29d49eff9507afeebefab7eeb2ca667fb4"
 dependencies = [
  "pem",
  "ring",

{mq_bridge_py_basic-0.4.0 → mq_bridge_py_basic-0.4.1}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.0"
+version = "0.4.1"
 
 [workspace.dependencies]
 anyhow = "1.0"

{mq_bridge_py_basic-0.4.0 → mq_bridge_py_basic-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.4.0
+Version: 0.4.1
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.4.0 → mq_bridge_py_basic-0.4.1}/benches/performance_bench.rs

@@ -16,6 +16,7 @@ use mq_bridge::test_utils::{
 };
 
 const PERF_TEST_MESSAGE_COUNT: usize = 1000;
+const GRPC_SERVER_MESSAGE_COUNT: usize = 25_600;
 
 #[allow(unused)]
 #[cfg(feature = "rustls")]
@@ -1244,6 +1245,7 @@ fn performance_benchmarks(c: &mut Criterion) {
     // report it as both write and read.
     #[cfg(feature = "grpc")]
     if mq_bridge::test_utils::should_run_benchmark("grpc_server") {
+        group.throughput(Throughput::Elements(GRPC_SERVER_MESSAGE_COUNT as u64));
         group.bench_function("grpc_server_batch", |b| {
             b.to_async(&rt).iter_custom(|iters| async move {
                 let (publisher, drain) = grpc_server_helper::setup_coupled().await;
@@ -1253,14 +1255,14 @@ fn performance_benchmarks(c: &mut Criterion) {
                     total += mq_bridge::test_utils::measure_write_performance(
                         "grpc_server_batch",
                         std::sync::Arc::clone(&publisher),
-                        PERF_TEST_MESSAGE_COUNT,
+                        GRPC_SERVER_MESSAGE_COUNT,
                         PERF_TEST_CONCURRENCY,
                     )
                     .await;
                 }
                 grpc_server_helper::finish_drain(drain).await;
                 let msgs_per_sec =
-                    (iters as f64 * PERF_TEST_MESSAGE_COUNT as f64) / total.as_secs_f64();
+                    (iters as f64 * GRPC_SERVER_MESSAGE_COUNT as f64) / total.as_secs_f64();
                 let mut results = BENCH_RESULTS.lock().await;
                 let stats = results.entry("grpc_server".to_string()).or_default();
                 stats.write_performance = msgs_per_sec;
@@ -1281,14 +1283,14 @@ fn performance_benchmarks(c: &mut Criterion) {
                     total += mq_bridge::test_utils::measure_single_write_performance(
                         "grpc_server_single",
                         std::sync::Arc::clone(&publisher),
-                        PERF_TEST_MESSAGE_COUNT,
+                        GRPC_SERVER_MESSAGE_COUNT,
                         PERF_TEST_CONCURRENCY,
                     )
                     .await;
                 }
                 grpc_server_helper::finish_drain(drain).await;
                 let msgs_per_sec =
-                    (iters as f64 * PERF_TEST_MESSAGE_COUNT as f64) / total.as_secs_f64();
+                    (iters as f64 * GRPC_SERVER_MESSAGE_COUNT as f64) / total.as_secs_f64();
                 let mut results = BENCH_RESULTS.lock().await;
                 let stats = results.entry("grpc_server".to_string()).or_default();
                 stats.single_write_performance = msgs_per_sec;

{mq_bridge_py_basic-0.4.0 → mq_bridge_py_basic-0.4.1}/src/route.rs

@@ -58,9 +58,23 @@ pub enum RouteOutcome {
 struct OutcomeGuard {
     outcome: Arc<RwLock<Option<RouteOutcome>>>,
     status: Arc<RwLock<EndpointStatus>>,
+    drops: Arc<RwLock<DropReport>>,
     resolved: Option<RouteOutcome>,
 }
 
+/// Messages the route discarded because the sink rejected them permanently and
+/// no DLQ was configured.
+///
+/// Accumulated by the route task and published by [`OutcomeGuard`] as the route
+/// ends, rather than written to the status directly: the reconnect loop clears
+/// `EndpointStatus::error` when a pass reports ready, so a drop recorded inline
+/// races that reset and is usually erased before anyone can read it.
+#[derive(Default)]
+pub(crate) struct DropReport {
+    count: u64,
+    last_cause: Option<String>,
+}
+
 impl OutcomeGuard {
     fn set(&mut self, outcome: RouteOutcome) {
         self.resolved = Some(outcome);
@@ -77,6 +91,22 @@ impl Drop for OutcomeGuard {
             s.error = Some("route task panicked or was aborted".to_string());
             RouteOutcome::Failed
         });
+        // A route that discarded data did not run clean, whatever its outcome.
+        // Never overwrite a recorded failure cause: that one explains why the
+        // route stopped, which is the more urgent of the two.
+        {
+            let drops = recover_read_lock(&self.drops, "route_drop_report");
+            if drops.count > 0 {
+                let mut s = recover_write_lock(&self.status, "route_handle_status");
+                if s.error.is_none() {
+                    s.error = Some(format!(
+                        "dropped {} message(s): sink rejected them permanently and no dlq middleware is configured: {}",
+                        drops.count,
+                        drops.last_cause.as_deref().unwrap_or("unknown cause")
+                    ));
+                }
+            }
+        }
         *recover_write_lock(&self.outcome, "route_handle_outcome") = Some(outcome);
     }
 }
@@ -350,6 +380,24 @@ fn report_route_error(err_tx: &Sender<anyhow::Error>, err: anyhow::Error, contex
     }
 }
 
+/// Records a batch the route discarded after a permanent sink rejection.
+///
+/// Dropping is deliberate — it is what stops a poison message from wedging the
+/// route — but `err_tx` only carries route-*stopping* errors, so without this a
+/// drop is invisible: the route resolves to [`RouteOutcome::Completed`] with
+/// `error: None` and the caller cannot tell a clean run from one that threw data
+/// away. [`OutcomeGuard`] publishes the tally when the route ends.
+fn record_dropped_messages(
+    drops: Option<&Arc<RwLock<DropReport>>>,
+    dropped: usize,
+    cause: &PublisherError,
+) {
+    let Some(drops) = drops else { return };
+    let mut report = recover_write_lock(drops, "route_drop_report");
+    report.count += dropped as u64;
+    report.last_cause = Some(cause.to_string());
+}
+
 struct BatchScratch {
     message_ids: Vec<u128>,
     request_ids: HashSet<u128>,
@@ -388,6 +436,7 @@ async fn send_batch_and_commit(
     commit_semaphore: Option<&Arc<tokio::sync::Semaphore>>,
     commit_tasks: &mut JoinSet<()>,
     scratch: &mut BatchScratch,
+    drops: Option<&Arc<RwLock<DropReport>>>,
 ) -> anyhow::Result<()> {
     let batch_len = messages.len();
     scratch.fill_from(&messages);
@@ -447,6 +496,25 @@ async fn send_batch_and_commit(
                     failed.len(),
                     first_err
                 );
+                // Non-retryable entries in a mixed batch are Ack'ed (dropped) below,
+                // so account for them here before the transient path returns.
+                if !has_dlq_middleware {
+                    let mut dropped = 0usize;
+                    let mut cause = None;
+                    for (msg, e) in &failed {
+                        if matches!(e, PublisherError::NonRetryable(_)) {
+                            error!(
+                                "Dropping message (ID: {:032x}) due to non-retryable error: {}",
+                                msg.message_id, e
+                            );
+                            dropped += 1;
+                            cause.get_or_insert(e);
+                        }
+                    }
+                    if let Some(cause) = cause {
+                        record_dropped_messages(drops, dropped, cause);
+                    }
+                }
                 let dispositions = map_responses_to_dispositions(
                     &scratch.message_ids,
                     responses,
@@ -473,6 +541,11 @@ async fn send_batch_and_commit(
                     msg.message_id, e
                 );
             }
+            if !has_dlq_middleware {
+                if let Some((_, first)) = failed.first() {
+                    record_dropped_messages(drops, failed.len(), first);
+                }
+            }
             let err_tx = err_tx.clone();
             let dispositions = map_responses_to_dispositions(
                 &scratch.message_ids,
@@ -504,6 +577,17 @@ async fn send_batch_and_commit(
             debug!("Failure commit result: {:?}", commit_result);
             if non_retryable && !has_dlq_middleware {
                 commit_result?;
+                // Acking a permanently-rejected batch keeps the route from
+                // re-reading the same poison messages forever, but the data is
+                // gone. Record it: a drop that leaves no trace is how a route
+                // reports a clean success while having discarded messages.
+                for id in scratch.message_ids.iter() {
+                    error!(
+                        "Dropping message (ID: {:032x}) due to non-retryable error: {}",
+                        id, e
+                    );
+                }
+                record_dropped_messages(drops, batch_len, &e);
                 Ok(())
             } else {
                 Err(e.into())
@@ -798,9 +882,12 @@ impl Route {
         let status_loop = Arc::clone(&status);
         // Terminal outcome cell, published by `OutcomeGuard` as the task exits.
         let outcome = Arc::new(RwLock::new(None::<RouteOutcome>));
+        // Tally of discarded messages, published alongside the terminal outcome.
+        let drops = Arc::new(RwLock::new(DropReport::default()));
         let mut outcome_guard = OutcomeGuard {
             outcome: Arc::clone(&outcome),
             status: Arc::clone(&status),
+            drops: Arc::clone(&drops),
             resolved: None,
         };
 
@@ -826,9 +913,15 @@ impl Route {
                 let (iter_ready_tx, iter_ready_rx) = bounded(1);
 
                 // The actual route logic is in `run_until_err`.
+                let drops_run = Arc::clone(&drops);
                 let mut run_task = tokio::spawn(async move {
                     route_arc
-                        .run_until_err(&name_arc, Some(internal_shutdown_rx), Some(iter_ready_tx))
+                        .run_until_err_reporting_to(
+                            &name_arc,
+                            Some(internal_shutdown_rx),
+                            Some(iter_ready_tx),
+                            Some(&drops_run),
+                        )
                         .await
                 });
 
@@ -1007,6 +1100,21 @@ impl Route {
         name: &str,
         shutdown_rx: Option<async_channel::Receiver<()>>,
         ready_tx: Option<Sender<()>>,
+    ) -> anyhow::Result<bool> {
+        self.run_until_err_reporting_to(name, shutdown_rx, ready_tx, None)
+            .await
+    }
+
+    /// [`run_until_err`](Self::run_until_err) with somewhere to report dropped
+    /// messages. A deployed route passes its status cell so that a permanent
+    /// sink rejection is still visible after the route completes; a caller that
+    /// drives a `Route` directly has no such cell and passes `None`.
+    pub(crate) async fn run_until_err_reporting_to(
+        &self,
+        name: &str,
+        shutdown_rx: Option<async_channel::Receiver<()>>,
+        ready_tx: Option<Sender<()>>,
+        drops: Option<&Arc<RwLock<DropReport>>>,
     ) -> anyhow::Result<bool> {
         let (_internal_shutdown_tx, internal_shutdown_rx) = bounded(1);
         let shutdown_rx = shutdown_rx.unwrap_or(internal_shutdown_rx);
@@ -1021,9 +1129,11 @@ impl Route {
             return result;
         }
         if self.options.concurrency == 1 {
-            self.run_sequentially(name, shutdown_rx, ready_tx).await
+            self.run_sequentially(name, shutdown_rx, ready_tx, drops)
+                .await
         } else {
-            self.run_concurrently(name, shutdown_rx, ready_tx).await
+            self.run_concurrently(name, shutdown_rx, ready_tx, drops)
+                .await
         }
     }
 
@@ -1033,6 +1143,7 @@ impl Route {
         name: &str,
         shutdown_rx: async_channel::Receiver<()>,
         ready_tx: Option<Sender<()>>,
+        drops: Option<&Arc<RwLock<DropReport>>>,
     ) -> anyhow::Result<bool> {
         let source_metadata_required = output_requires_source_metadata(name, &self.output)?;
         let publisher = create_publisher_from_route(name, &self.output).await?;
@@ -1129,6 +1240,7 @@ impl Route {
                         commit_semaphore.as_ref(),
                         &mut commit_tasks,
                         &mut batch_scratch,
+                        drops,
                     )
                     .await
                     {
@@ -1174,6 +1286,7 @@ impl Route {
         name: &str,
         shutdown_rx: async_channel::Receiver<()>,
         ready_tx: Option<Sender<()>>,
+        drops: Option<&Arc<RwLock<DropReport>>>,
     ) -> anyhow::Result<bool> {
         let source_metadata_required = output_requires_source_metadata(name, &self.output)?;
         let publisher = create_publisher_from_route(name, &self.output).await?;
@@ -1224,6 +1337,8 @@ impl Route {
             let has_retry_middleware = self.output.has_retry_middleware();
             let has_dlq_middleware = self.output.has_dlq_middleware();
             let batch_size = self.options.batch_size;
+            // Owned per worker: the borrow cannot outlive this loop iteration.
+            let drops = drops.cloned();
             join_set.spawn(async move {
                 debug!("Starting worker {}", i);
                 let mut batch_scratch = BatchScratch::with_capacity(batch_size);
@@ -1240,6 +1355,7 @@ impl Route {
                         commit_semaphore.as_ref(),
                         &mut commit_tasks,
                         &mut batch_scratch,
+                        drops.as_ref(),
                     )
                     .await
                     {
@@ -3953,6 +4069,61 @@ mod tests {
         assert_eq!(attempts.values().filter(|&&c| c == 1).count(), 2);
     }
 
+    /// A sink that rejects permanently with no DLQ configured drops the batch so
+    /// the route is not wedged re-reading a poison message — but the drop must
+    /// not be silent. The route still completes; it just carries the cause, so a
+    /// caller polling only `outcome` cannot mistake a discarded batch for a
+    /// clean delivery.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_dropped_poison_batch_is_recorded_on_the_route_status() {
+        let unique_id = fast_uuid_v7::gen_id().to_string();
+        let in_topic = format!("drop_in_{}", unique_id);
+        let out_topic = format!("drop_out_{}", unique_id);
+        let input = Endpoint::new_memory(&in_topic, 10);
+
+        let mut output = Endpoint::new_memory(&out_topic, 10);
+        output.middlewares = vec![Middleware::RandomPanic(RandomPanicMiddleware {
+            mode: FaultMode::JsonFormatError,
+            trigger_on_message: None,
+            enabled: true,
+            ..Default::default()
+        })];
+
+        let input_ch = input.channel().unwrap();
+        input_ch.send_message("poison".into()).await.unwrap();
+
+        let route = Route::new(input, output)
+            .with_fault_injection(true)
+            .with_exit_on_empty(true);
+        let handle = route.run("test_dropped_poison_recorded").await.unwrap();
+
+        let outcome = tokio::time::timeout(Duration::from_secs(10), async {
+            loop {
+                if let Some(outcome) = handle.outcome() {
+                    return outcome;
+                }
+                tokio::time::sleep(Duration::from_millis(20)).await;
+            }
+        })
+        .await
+        .expect("the drain finishes rather than retrying the poison batch forever");
+
+        assert_eq!(
+            outcome,
+            RouteOutcome::Completed,
+            "dropping the batch lets the drain finish"
+        );
+        let error = handle.status().error.unwrap_or_default();
+        assert!(
+            error.contains("dropped 1 message"),
+            "the discarded batch is reported, not silently swallowed: {error:?}"
+        );
+        assert!(
+            error.contains("JSON format error"),
+            "the reported cause names the underlying rejection: {error:?}"
+        );
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     async fn test_route_dlq_integration() {
         // Setup: Input -> [Panic(Disconnect) -> Retry -> DLQ] -> Output