mq-bridge-py-basic 0.4.3__tar.gz → 0.4.4__tar.gz

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/Cargo.lock

@@ -3348,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3431,7 +3431,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3445,7 +3445,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3487,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.4.3"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "async-trait",

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/Cargo.toml

@@ -16,7 +16,7 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.3"
+version = "0.4.4"
 
 [workspace.dependencies]
 anyhow = "1.0"

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.4.3
+Version: 0.4.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/src/endpoints/file/mod.rs

@@ -2056,11 +2056,20 @@ impl FileConsumer {
     }
 
     pub async fn new(config: &FileConfig) -> anyhow::Result<Self> {
+        Self::new_with_source_metadata(config, config.source_metadata).await
+    }
+
+    /// `source_metadata` is the effective flag: the route enables it for an idempotent
+    /// output even when the input config leaves it unset.
+    pub async fn new_with_source_metadata(
+        config: &FileConfig,
+        source_metadata: bool,
+    ) -> anyhow::Result<Self> {
         let startup_open_error = probe_source_path(&config.path)?;
         let mut consumer = Self::new_backend(config).await?;
         consumer.path = config.path.clone();
         consumer.startup_open_error = startup_open_error;
-        consumer.source_metadata = config.source_metadata;
+        consumer.source_metadata = source_metadata;
         // `consume` always reads from byte 0, so its record index is reproducible and two
         // runs deliberately produce the same names — that is what makes the sink idempotent.
         // `subscribe` starts at the current end and `group_subscribe` resumes at a stored
@@ -2068,7 +2077,7 @@ impl FileConsumer {
         // numbered. Without an epoch those names would collide and the sink would discard
         // the new records as already-covered; with one they stay distinct and ordered, at
         // the cost of cross-restart deduplication.
-        consumer.run_epoch = (config.source_metadata
+        consumer.run_epoch = (source_metadata
             && !matches!(&config.mode, None | Some(FileConsumerMode::Consume { .. })))
         .then(next_run_epoch);
         Ok(consumer)

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/src/endpoints/mod.rs

@@ -1017,6 +1017,10 @@ async fn run_http_inline_response_fast_path(
     Ok(true)
 }
 
+/// `_source_metadata` is honoured by the types [`supports_source_metadata`] admits and by the
+/// NATS and AMQP consumers, which stamp provenance that is not a replay position. The other
+/// branches ignore the flag. Startup only rejects an input for missing replay positions when
+/// `source_metadata_required` is set.
 async fn create_base_consumer(
     route_name: &str,
     endpoint: &Endpoint,
@@ -1092,7 +1096,9 @@ async fn create_base_consumer(
                 redis_streams::RedisStreamsConsumer::new(&config).await?,
             ))
         }
-        EndpointType::File(cfg) => Ok(boxed(file::FileConsumer::new(cfg).await?)),
+        EndpointType::File(cfg) => Ok(boxed(
+            file::FileConsumer::new_with_source_metadata(cfg, _source_metadata).await?,
+        )),
         #[cfg(feature = "object-store")]
         EndpointType::ObjectStore(cfg) => {
             Ok(boxed(object_store::ObjectStoreConsumer::new(cfg).await?))
@@ -2107,6 +2113,82 @@ mod tests {
             .contains("requires an input that carries a replay position"));
     }
 
+    /// The route derives `source_metadata` from an idempotent output, so a plain `file`
+    /// input must be stamped without the caller setting the flag on the source config.
+    #[tokio::test]
+    async fn file_input_is_stamped_when_the_output_is_idempotent() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("orders.jsonl");
+        std::fs::write(&input, "{\"id\":1}\n{\"id\":2}\n").unwrap();
+
+        let file = Endpoint::new(EndpointType::File(crate::models::FileConfig::new(
+            input.to_string_lossy(),
+        )));
+        let mut consumer =
+            create_consumer_from_route_with_source_metadata("idempotent-route", &file, true)
+                .await
+                .unwrap();
+
+        let batch = consumer.receive_batch(10).await.unwrap();
+        assert_eq!(batch.messages.len(), 2, "file input produced no rows");
+        for (index, message) in batch.messages.iter().enumerate() {
+            assert_eq!(
+                message
+                    .metadata
+                    .get("mqb.src.file_path")
+                    .map(String::as_str),
+                Some(input.to_string_lossy().as_ref())
+            );
+            // The record index is the replay position, so it has to be the row's own index.
+            assert_eq!(
+                message
+                    .metadata
+                    .get("mqb.src.file_record")
+                    .map(String::as_str),
+                Some(index.to_string().as_str())
+            );
+            // `consume` reproduces its record index, so it carries no epoch.
+            assert!(!message.metadata.contains_key("mqb.src.file_epoch"));
+            assert!(crate::support::source_ranges::SourcePosition::from_message(message).is_ok());
+        }
+    }
+
+    /// The epoch reads the same derived flag: a `group_subscribe` input restarts its record
+    /// index, so without one the second run's names would collide with the first run's.
+    #[tokio::test]
+    async fn derived_source_metadata_also_gives_group_subscribe_a_run_epoch() {
+        let dir = tempfile::tempdir().unwrap();
+        let input = dir.path().join("orders.jsonl");
+        std::fs::write(&input, "{\"id\":1}\n").unwrap();
+
+        let file = Endpoint::new(EndpointType::File(crate::models::FileConfig {
+            mode: Some(crate::models::FileConsumerMode::GroupSubscribe {
+                group_id: "derived-epoch".to_string(),
+                read_from_tail: false,
+            }),
+            ..crate::models::FileConfig::new(input.to_string_lossy())
+        }));
+
+        // Two runs of the same input: both restart the record index at 0, so only the epoch
+        // keeps the second run's records from being named like the first run's.
+        let mut positions = Vec::new();
+        for _ in 0..2 {
+            let mut consumer =
+                create_consumer_from_route_with_source_metadata("idempotent-route", &file, true)
+                    .await
+                    .unwrap();
+            let batch = consumer.receive_batch(10).await.unwrap();
+            let first = batch.messages.first().expect("file input produced no rows");
+            assert!(first.metadata.contains_key("mqb.src.file_epoch"));
+            positions
+                .push(crate::support::source_ranges::SourcePosition::from_message(first).unwrap());
+        }
+
+        assert_ne!(positions[0].source, positions[1].source);
+        // A later run reads later records, so its objects must sort after the earlier run's.
+        assert!(positions[0].source < positions[1].source);
+    }
+
     #[tokio::test]
     async fn test_fanout_publisher_integration() {
         let ep1 = Endpoint::new_memory("fanout_1", 10);

{mq_bridge_py_basic-0.4.3 → mq_bridge_py_basic-0.4.4}/src/endpoints/object_store.rs

@@ -31,7 +31,7 @@ use crate::endpoints::file::{encode_record, parse_delimiter, parse_message};
 use crate::models::{Compression, FileFormat, ObjectStoreConfig};
 #[cfg(feature = "encryption")]
 use crate::support::crypto::Crypto;
-use crate::support::source_ranges::{finalized_name, CoveredRanges};
+use crate::support::source_ranges::{finalized_name, parse_finalized_name, CoveredRanges};
 use crate::traits::{
     BoxFuture, ConsumerError, MessageConsumer, MessageDisposition, MessagePublisher,
     PublisherError, ReceivedBatch, SentBatch,
@@ -127,18 +127,53 @@ async fn recover_idempotent_object_ranges(
     base: &ObjPath,
     extension: &str,
 ) -> anyhow::Result<CoveredRanges> {
-    let mut names = Vec::new();
+    // Parsed as the listing streams: a bucket holding a long history of parts would otherwise
+    // materialise every key at once, and only the merged ranges are worth keeping.
+    let mut covered = CoveredRanges::default();
     let mut stream = store.list(Some(base));
     while let Some(meta) = stream.next().await {
-        let key = meta?.location.to_string();
-        if let Some(name) = key.rsplit('/').next() {
-            names.push(name.to_string());
+        let location = meta?.location;
+        // The listing is recursive, but this sink writes its parts flat under `base`. A
+        // parseable name below a nested prefix is another sink's part, and counting it here
+        // would mark ranges covered that were never written to this prefix.
+        let Some(mut parts) = location.prefix_match(base) else {
+            continue;
+        };
+        let Some(name) = parts.next() else {
+            continue;
+        };
+        if parts.next().is_some() {
+            continue;
+        }
+        if let Some((source, start, end)) = parse_finalized_name(name.as_ref(), extension) {
+            // `parse_finalized_name` already guarantees a valid range.
+            covered.insert(source, start, end)?;
         }
     }
-    Ok(CoveredRanges::from_finalized_names(
-        names.iter().map(String::as_str),
-        extension,
-    ))
+    Ok(covered)
+}
+
+/// Classifies a failed PUT. A denied credential or a store that does not implement the
+/// requested mode does not become writable by trying again, so those stop the route with the
+/// reason instead of retrying forever. `PutMode::Create` on a store with conditional put
+/// disabled lands here as `NotImplemented`, which is how an idempotent sink reports that the
+/// bucket cannot give it the write-once guarantee it is built on.
+fn classify_put_error(error: ObjectStoreError, context: String) -> PublisherError {
+    let permanent = matches!(
+        error,
+        ObjectStoreError::NotImplemented { .. }
+            | ObjectStoreError::NotSupported { .. }
+            | ObjectStoreError::PermissionDenied { .. }
+            | ObjectStoreError::Unauthenticated { .. }
+            | ObjectStoreError::InvalidPath { .. }
+            | ObjectStoreError::UnknownConfigurationKey { .. }
+    );
+    let error = anyhow!(error).context(context);
+    if permanent {
+        PublisherError::NonRetryable(error)
+    } else {
+        PublisherError::Retryable(error)
+    }
 }
 
 /// Splits a fetched object into record slices on `delimiter`, dropping a trailing empty
@@ -322,7 +357,20 @@ impl ObjectStorePublisher {
         &self,
         messages: Vec<CanonicalMessage>,
     ) -> Result<SentBatch, PublisherError> {
-        let mut covered = self.covered_ranges.lock().await;
+        // The lock guards the shared range map, not the work done with it. Held across the
+        // encode and the PUT it would serialise the whole worker pool onto one in-flight
+        // object — a sink that publishes sequentially, which is what naming by source range
+        // exists to make unnecessary. A snapshot is a handful of merged ranges per partition,
+        // so cloning it costs far less than what it lets run concurrently.
+        //
+        // The route enqueues each message once, so batches in flight together hold disjoint
+        // ranges and cannot race here at all. A re-sent batch repeats a range exactly, and an
+        // identical range is an identical name, which `PutMode::Create` turns into a no-op.
+        // Only a source that redelivers offsets still in flight — a Kafka rebalance mid-batch —
+        // could produce two runs that overlap *partially*; those name two different objects and
+        // the overlap would be written twice. Claiming the range under the lock before the PUT
+        // would close that window, at the cost of unwinding the claim on every failure path.
+        let covered = self.covered_ranges.lock().await.clone();
         let runs = covered
             .uncovered_runs(messages)
             .map_err(PublisherError::NonRetryable)?;
@@ -333,6 +381,10 @@ impl ObjectStorePublisher {
             let mut body = Vec::new();
             for mut message in run.messages {
                 message.strip_source_metadata();
+                // Unlike the ordinary path this cannot report `Partial` and hand one bad record
+                // to a DLQ: the object is named for the whole range, so skipping a record inside
+                // it would mark offsets covered that were never written. The batch fails whole
+                // instead. Runs already PUT stay written and are skipped on the resend.
                 let bytes = encode_record(&message, &self.format)
                     .map_err(|error| PublisherError::NonRetryable(anyhow!(error)))?;
                 body.extend_from_slice(&bytes);
@@ -354,12 +406,15 @@ impl ObjectStorePublisher {
             {
                 Ok(_) | Err(ObjectStoreError::AlreadyExists { .. }) => {}
                 Err(error) => {
-                    return Err(PublisherError::Retryable(
-                        anyhow!(error).context(format!("object-store put idempotent '{key}'")),
+                    return Err(classify_put_error(
+                        error,
+                        format!("object-store put idempotent '{key}'"),
                     ));
                 }
             }
-            covered
+            self.covered_ranges
+                .lock()
+                .await
                 .insert(run.source, run.start, run.end)
                 .map_err(PublisherError::NonRetryable)?;
         }
@@ -421,9 +476,7 @@ impl MessagePublisher for ObjectStorePublisher {
         self.store
             .put(&key, PutPayload::from(body))
             .await
-            .map_err(|e| {
-                PublisherError::Retryable(anyhow!(e).context(format!("object-store put '{key}'")))
-            })?;
+            .map_err(|e| classify_put_error(e, format!("object-store put '{key}'")))?;
         trace!(key = %key, "Wrote object to object store");
         if failed.is_empty() {
             Ok(SentBatch::Ack)
@@ -1010,6 +1063,119 @@ mod tests {
         message
     }
 
+    /// A store whose `put_opts` parks on a barrier, so a put can only complete once as many
+    /// puts are in flight as the barrier expects. A publisher that serialises its writes
+    /// never gets there and hangs, which is what turns "slower" into a failing test.
+    #[derive(Debug)]
+    struct BarrierStore {
+        inner: InMemory,
+        puts: tokio::sync::Barrier,
+    }
+
+    impl BarrierStore {
+        fn new(concurrent_puts: usize) -> Self {
+            Self {
+                inner: InMemory::new(),
+                puts: tokio::sync::Barrier::new(concurrent_puts),
+            }
+        }
+    }
+
+    impl std::fmt::Display for BarrierStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "BarrierStore")
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for BarrierStore {
+        async fn put_opts(
+            &self,
+            location: &ObjPath,
+            payload: PutPayload,
+            opts: PutOptions,
+        ) -> object_store::Result<object_store::PutResult> {
+            self.puts.wait().await;
+            self.inner.put_opts(location, payload, opts).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &ObjPath,
+            opts: object_store::PutMultipartOptions,
+        ) -> object_store::Result<Box<dyn object_store::MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+
+        async fn get_opts(
+            &self,
+            location: &ObjPath,
+            options: object_store::GetOptions,
+        ) -> object_store::Result<object_store::GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+
+        fn delete_stream(
+            &self,
+            locations: futures::stream::BoxStream<'static, object_store::Result<ObjPath>>,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<ObjPath>> {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(
+            &self,
+            prefix: Option<&ObjPath>,
+        ) -> futures::stream::BoxStream<'static, object_store::Result<object_store::ObjectMeta>>
+        {
+            self.inner.list(prefix)
+        }
+
+        async fn list_with_delimiter(
+            &self,
+            prefix: Option<&ObjPath>,
+        ) -> object_store::Result<object_store::ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy_opts(
+            &self,
+            from: &ObjPath,
+            to: &ObjPath,
+            options: object_store::CopyOptions,
+        ) -> object_store::Result<()> {
+            self.inner.copy_opts(from, to, options).await
+        }
+    }
+
+    /// The idempotent path must not trade the worker pool for its ordering guarantee. Naming
+    /// objects by source range is what makes concurrent writes safe, so the range bookkeeping
+    /// may only be locked around the map itself — never around the encode and the PUT, which
+    /// would leave an ordered sink publishing one object at a time.
+    #[tokio::test]
+    async fn idempotent_sends_are_not_serialised_by_the_covered_range_lock() {
+        let store = Arc::new(BarrierStore::new(2));
+        let mut publisher = test_publisher(store);
+        publisher.idempotency = true;
+        let other = publisher.clone();
+
+        // Disjoint offset runs, so both batches genuinely write and neither is skipped.
+        let first = tokio::spawn(async move {
+            publisher
+                .send_batch_idempotent(vec![kafka_message(0)])
+                .await
+        });
+        let second =
+            tokio::spawn(async move { other.send_batch_idempotent(vec![kafka_message(1)]).await });
+
+        let sends = async {
+            first.await.unwrap().unwrap();
+            second.await.unwrap().unwrap();
+        };
+        tokio::time::timeout(Duration::from_secs(10), sends)
+            .await
+            .expect("both puts must be in flight at once; the covered-range lock is being held across the put");
+    }
+
     /// The guarantee the idempotent sink exists to make, and the one the whole ordering
     /// thread turns on: object order is *source* order even when batches are written
     /// concurrently and out of order. Dispatches the batches in reverse and lets them race,
@@ -1109,6 +1275,44 @@ mod tests {
         );
     }
 
+    /// `list` is recursive, so a sink whose base is a *sub*prefix of ours shows up in our
+    /// listing with a perfectly parseable part name. Counting it would mark that range
+    /// covered here and silently drop the records that were never written under our prefix.
+    #[tokio::test]
+    async fn recovery_ignores_parseable_parts_under_a_nested_prefix() {
+        let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
+        let nested = ObjPath::from(format!(
+            "data/other/{}",
+            part_key(0, 1, "jsonl").filename().unwrap()
+        ));
+        store
+            .put(&nested, PutPayload::from("someone else's part"))
+            .await
+            .unwrap();
+
+        let base = ObjPath::from("data");
+        let covered = recover_idempotent_object_ranges(store.as_ref(), &base, "jsonl")
+            .await
+            .unwrap();
+
+        let mut publisher = test_publisher(store.clone());
+        publisher.idempotency = true;
+        publisher.covered_ranges = Arc::new(Mutex::new(covered));
+        publisher
+            .send_batch(vec![kafka_message(0), kafka_message(1)])
+            .await
+            .unwrap();
+
+        let written = store
+            .get(&part_key(0, 1, "jsonl"))
+            .await
+            .expect("the nested part must not count as coverage")
+            .bytes()
+            .await
+            .unwrap();
+        assert!(!written.is_empty());
+    }
+
     #[tokio::test]
     async fn idempotent_sink_does_not_overwrite_an_existing_range_object() {
         let store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());