mq-bridge-py-basic 0.4.2__tar.gz → 0.4.4__tar.gz

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/Cargo.lock

@@ -1900,9 +1900,9 @@ dependencies = [
 
 [[package]]
 name = "fast-uuid-v7"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65d7a30239eba18ae6a60dc3c3df98af18f00bba69d2b1d5f2816a831e83bff3"
+checksum = "dc759201eb9b09178fb2174087a451d19fd46a8b8b4f5f6103de83b9f5949f52"
 dependencies = [
  "rand 0.10.2",
 ]
@@ -3348,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3431,7 +3431,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3445,7 +3445,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3487,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.4.2"
+version = "0.4.4"
 dependencies = [
  "anyhow",
  "async-trait",

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/Cargo.toml

@@ -16,13 +16,13 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.2"
+version = "0.4.4"
 
 [workspace.dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
 bytes = "1"
-fast-uuid-v7 = "0.1.8"
+fast-uuid-v7 = "0.1.9"
 serde = "1.0"
 serde_json = { version = "1.0", features = ["raw_value"] }
 serde_yaml_ng = "0.10"

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.4.2
+Version: 0.4.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/docs/DELIVERY.md

@@ -22,7 +22,7 @@ sink writes data.
 | MongoDB | `id_field` set to a payload field or replay-stable template such as `${metadata:mqb.id}` |
 | PostgreSQL / SQLite | `sqlx.insert_query` uses a unique key with `ON CONFLICT` |
 | MySQL / MariaDB | `sqlx.insert_query` uses a unique key with `ON DUPLICATE KEY` |
-| File / object store | `idempotency: true` with a replayable Kafka or Postgres CDC source |
+| File / object store | `idempotency: true` with a replayable Kafka, Postgres CDC, SQL cursor, MongoDB CDC or file source |
 
 For a source without stable identity, derive one once and consume it at the sink:
 
@@ -143,7 +143,7 @@ one identity to every message missing the field, so it is dropped instead.
 | `mongodb` | Unique `_id` index; dup-key (11000) is treated as an idempotent success | `id_field` |
 | `sqlx` (PostgreSQL / MySQL / SQLite) | The table's own `UNIQUE`/`PRIMARY KEY` | `ON CONFLICT` / `ON DUPLICATE KEY` in `insert_query` |
 | `clickhouse` | `ReplacingMergeTree` collapses at merge time | Table DDL — no `mq-bridge` config |
-| `file`, `object_store` | Deterministic part names + covered-range recovery | `idempotency: true` (needs a replayable source) |
+| `file`, `object_store` | Deterministic, sortable part names + covered-range recovery | `idempotency: true` (needs a replayable source) |
 | `kafka` | `enable.idempotence` dedups **producer retries within one session** — this is *not* exactly-once semantics | On by default |
 | `nats`, `amqp`, `mqtt`, `redis_streams`, `aws`, `ibm_mq`, `zeromq` | None | Deduplicate at the next consumer instead |
 
@@ -359,6 +359,13 @@ but before the branch's downstream send committed, the replay hits a duplicate k
 
 ### Files & object storage — `idempotency`
 
+> **Check first whether you need any of this.** If your records already carry a business key — an
+> `id` field in the payload, or `mqb.id` from the [`id` middleware](#giving-a-source-an-identity) —
+> then a key-addressed sink (Mongo `id_field`, SQL `ON CONFLICT`) already gives you effective
+> exactly-once, and the order objects happen to land in does not matter. Positional naming below is
+> for the case where the *sink itself* has to recognise a replay, or where a downstream reader
+> depends on replay **order**. Turning it on when you do not need it only adds restrictions.
+
 A filesystem has no unique constraint, so the `file` and `object_store` sinks get replay safety a
 different way: **deterministic names plus covered-range recovery**. Set `idempotency: true` and the
 sink stops writing UUID-named objects. Instead it groups each batch into runs of consecutive source
@@ -371,7 +378,7 @@ kafka_to_s3:
   output:
     object_store:
       url: "s3://my-bucket/orders"
-      idempotency: true          # → s3://my-bucket/orders/part-orders-0-0-1023.jsonl
+      idempotency: true          # → part-orders-<partition>-<start>-<end>.jsonl (zero-padded)
 ```
 
 On startup the sink lists what is already there, parses the ranges out of the part names, and drops
@@ -381,13 +388,58 @@ checkpoint protocol. Local files stage to a temp name, `fsync`, then `rename` (a
 filesystem); object stores have no atomic rename, so a single PUT under the final name *is* the
 commit.
 
-This needs a replayable source position, which today means **Kafka** (topic/partition/offset) or
-**`postgres_cdc`** (commit LSN plus in-transaction ordinal; `sqlx` with `publication` maps onto the
-same consumer). A route with an idempotent output turns `source_metadata` on for its input
-automatically; set it explicitly only when you want the `mqb.src.*` keys for something else. Any
-other input is rejected when the route starts. NATS, AMQP and MongoDB CDC also accept
-`source_metadata` and emit provenance keys, but a subject or routing key is not a replayable offset,
-so they cannot drive an idempotent sink.
+This needs a replayable source position. Today that means:
+
+| Source | Position | Restriction |
+| --- | --- | --- |
+| `kafka` | topic / partition / offset | — |
+| `postgres_cdc` | commit LSN + in-transaction ordinal | `sqlx` with `publication` maps onto the same consumer |
+| `mongodb` | cluster time + ordinal; the initial snapshot uses its `_id` scan index | `capture_new` / `capture_all` only — `consumer` and `snapshot` have no position |
+| `file` | record index within the file | all modes; only `consume` deduplicates across runs (see below) |
+| `sqlx` | the `cursor_column` value | cursor polling only; the column must be a unique, non-negative integer |
+
+A route with an idempotent output turns `source_metadata` on for its input automatically; set it
+explicitly only when you want the `mqb.src.*` keys for something else. Any other input is rejected
+when the route starts. NATS and AMQP also accept `source_metadata` and emit provenance keys, but a
+subject or routing key is not a replayable offset, so they cannot drive an idempotent sink.
+
+**Without `idempotency: true`, an `object_store` sink names objects in write order**, and key order
+is the only order a bucket has. At `concurrency: 1` that is source order; above it the name is minted
+inside the worker pool, so it is arrival order, and replaying a change stream through the bucket can
+reorder updates to the same key. The route logs a warning when it starts in that configuration.
+
+**Numbers inside a part name are zero-padded** so that ASCII sort equals numeric sort — a bucket is
+listed lexicographically and that listing order *is* the replay order. A bucket written by a version
+before this padding contains unpadded names; the two forms do not sort correctly against each other,
+so do not mix them in one prefix.
+
+**MongoDB `capture_all`** reads the collection, then streams changes. Both phases are numbered into
+the key so the snapshot always sorts ahead of the changes; a change can never replay before the
+document it modifies. The snapshot does not resume across runs — it re-scans from the start — but the
+numbering is deterministic, so a restart reproduces the same names and the covered-range recovery
+skips them. The exception is inserts or deletes landing in the collection between the two runs: those
+shift the numbering, and some documents are then written twice.
+
+**The `sqlx` cursor source** uses the `cursor_column` value itself as the position — the same shape
+Kafka Connect's S3 sink gives a Kafka offset. That only works for a unique, non-negative integer
+column: a text cursor pages fine but has no contiguous numeric order, and a repeated value would
+resolve two rows to one position and drop one of them. The reader rejects both at read time rather
+than naming records wrongly. Consecutive ids coalesce into one part file; gaps split it.
+
+**The `file` source** numbers records by their index in the file, not their byte offset (byte offsets
+are not consecutive, so every record would become its own part).
+
+That index repeats across runs only in `consume` mode, which always reads from byte 0 — and repeating
+is the point: a re-read produces the same names, so the sink recognises them as already covered and
+writes nothing. `subscribe` starts at the current end of the file and `group_subscribe` resumes at a
+stored byte offset, so their index restarts at 0 over records an earlier run already numbered. Those
+two modes therefore carry a **run epoch** in the object name.
+
+The epoch keeps names distinct and keeps runs in order (a later run reads later records, so its
+objects sort after the earlier run's). What it does not give you is deduplication across a restart:
+records re-read after a crash are written again under new names. That is ordinary at-least-once, and
+it is the honest guarantee for a source with no durable per-record position — these modes are
+allowed, not rejected, because that guarantee is fine for plenty of pipelines.
 
 For the `file` sink, `idempotency: true` changes what `path` means: it is the directory that receives
 the part files, not the file that is appended to. The sink creates it on startup, so pointing it at an

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/mq-bridge.schema.json

@@ -1012,13 +1012,18 @@
           "default": "normal"
         },
         "idempotency": {
-          "description": "Write replay-safe source ranges as immutable part files under this directory.\nRequires Kafka source metadata or postgres_cdc commit LSN plus transaction ordinal metadata.",
+          "description": "Write replay-safe source ranges as immutable part files under this directory.\nRequires a source that stamps a replayable position: kafka, postgres_cdc, mongodb CDC, sqlx or file.",
           "type": "boolean",
           "default": false
         },
         "path": {
           "description": "Path to the file, or to the directory holding the part files when `idempotency` is true.",
           "type": "string"
+        },
+        "source_metadata": {
+          "description": "(Consumer only) Include authoritative `mqb.src.file_*` source positions; only `consume` mode reproduces them across restarts. Defaults to false.",
+          "type": "boolean",
+          "default": false
         }
       },
       "anyOf": [
@@ -3217,6 +3222,11 @@
             "null"
           ]
         },
+        "source_metadata": {
+          "description": "(Consumer only) Include authoritative `mqb.src.sqlx_*` source positions; `cursor_column` must then be a unique integer. Defaults to false.",
+          "type": "boolean",
+          "default": false
+        },
         "table": {
           "description": "The table to interact with.",
           "type": "string"

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/mq_bridge/config.pyi

@@ -154,6 +154,7 @@ class FileConfig(TypedDict, total=False):
     format: FileFormat
     idempotency: bool
     path: Required[str]
+    source_metadata: bool
 
 
 class GrpcConfig(TypedDict, total=False):
@@ -476,6 +477,7 @@ class SqlxConfig(TypedDict, total=False):
     select_query: Optional[str]
     shared: Optional[bool]
     slot_name: Optional[str]
+    source_metadata: bool
     table: Required[str]
     test_before_acquire: Optional[bool]
     tls: TlsConfig

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/src/endpoints/file/mod.rs

@@ -19,7 +19,7 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::io::Seek;
 use std::path::Path;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex as StdMutex, Weak};
 use std::time::{Duration, SystemTime};
 use tokio::fs::{self, File, OpenOptions};
@@ -2013,24 +2013,73 @@ pub struct FileConsumer {
     /// A live tail keeps waiting for the file to appear; a drain (`exit_on_empty`)
     /// cannot, so it fails with this instead of blocking forever.
     startup_open_error: Option<String>,
+    /// Stamp `mqb.src.file_*` so an idempotent sink can name records in source order.
+    source_metadata: bool,
+    /// Index of the next record in this file.
+    next_record: u64,
+    /// Set only for the modes that do not start at byte 0, so their record indexes cannot
+    /// collide with a previous run's. `None` for `consume`, whose names must repeat.
+    run_epoch: Option<u64>,
     exit_on_empty: bool,
 }
 
+/// Hands out a strictly increasing run epoch. Seeded from wall-clock millis so epochs still
+/// sort across process restarts, but never repeats or goes backwards within one process —
+/// consumers built in the same millisecond, or across a clock step back, stay distinct.
+fn next_run_epoch() -> u64 {
+    static LAST: AtomicU64 = AtomicU64::new(0);
+    let now = SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_millis() as u64)
+        .unwrap_or(0);
+    let mut last = LAST.load(Ordering::Relaxed);
+    loop {
+        let epoch = now.max(last + 1);
+        match LAST.compare_exchange_weak(last, epoch, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return epoch,
+            Err(observed) => last = observed,
+        }
+    }
+}
+
 impl FileConsumer {
     fn wrap(backend: ConsumerBackend) -> Self {
         Self {
             backend,
             path: String::new(),
             startup_open_error: None,
+            source_metadata: false,
+            next_record: 0,
+            run_epoch: None,
             exit_on_empty: false,
         }
     }
 
     pub async fn new(config: &FileConfig) -> anyhow::Result<Self> {
+        Self::new_with_source_metadata(config, config.source_metadata).await
+    }
+
+    /// `source_metadata` is the effective flag: the route enables it for an idempotent
+    /// output even when the input config leaves it unset.
+    pub async fn new_with_source_metadata(
+        config: &FileConfig,
+        source_metadata: bool,
+    ) -> anyhow::Result<Self> {
         let startup_open_error = probe_source_path(&config.path)?;
         let mut consumer = Self::new_backend(config).await?;
         consumer.path = config.path.clone();
         consumer.startup_open_error = startup_open_error;
+        consumer.source_metadata = source_metadata;
+        // `consume` always reads from byte 0, so its record index is reproducible and two
+        // runs deliberately produce the same names — that is what makes the sink idempotent.
+        // `subscribe` starts at the current end and `group_subscribe` resumes at a stored
+        // byte offset, so their index restarts at 0 over records a previous run already
+        // numbered. Without an epoch those names would collide and the sink would discard
+        // the new records as already-covered; with one they stay distinct and ordered, at
+        // the cost of cross-restart deduplication.
+        consumer.run_epoch = (source_metadata
+            && !matches!(&config.mode, None | Some(FileConsumerMode::Consume { .. })))
+        .then(next_run_epoch);
         Ok(consumer)
     }
 
@@ -2344,6 +2393,37 @@ impl MessageConsumer for FileConsumer {
     // a cumulative byte offset (the max acked `file_offset`), so out-of-order
     // commits could advance the offset past un-acked messages and lose them.
     async fn receive_batch(&mut self, max_messages: usize) -> Result<ReceivedBatch, ConsumerError> {
+        let mut batch = self.receive_batch_inner(max_messages).await?;
+        if self.source_metadata {
+            for message in &mut batch.messages {
+                message
+                    .metadata
+                    .insert("mqb.src.file_path".to_string(), self.path.clone());
+                message.metadata.insert(
+                    "mqb.src.file_record".to_string(),
+                    self.next_record.to_string(),
+                );
+                if let Some(epoch) = self.run_epoch {
+                    message
+                        .metadata
+                        .insert("mqb.src.file_epoch".to_string(), epoch.to_string());
+                }
+                self.next_record += 1;
+            }
+        }
+        Ok(batch)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl FileConsumer {
+    async fn receive_batch_inner(
+        &mut self,
+        max_messages: usize,
+    ) -> Result<ReceivedBatch, ConsumerError> {
         // The path was unopenable at startup. A live tail waits for it (rotation,
         // a writer that hasn't created it yet), but a drain has nothing to wait
         // for, so re-probe and fail rather than block forever on a typo'd path.
@@ -2566,10 +2646,6 @@ impl MessageConsumer for FileConsumer {
             }
         }
     }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
 }
 
 /// Wraps a message body for the Json/Text file formats. Generic over the payload type

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/src/endpoints/file/tests.rs

@@ -450,8 +450,8 @@ async fn idempotent_file_sink_replays_only_uncovered_kafka_offsets_after_restart
         names,
         vec![
             ".stage-inflight".to_string(),
-            "part-orders-0-0-1.jsonl".to_string(),
-            "part-orders-0-2-2.jsonl".to_string(),
+            "part-orders-0000000000-00000000000000000000-00000000000000000001.jsonl".to_string(),
+            "part-orders-0000000000-00000000000000000002-00000000000000000002.jsonl".to_string(),
         ]
     );
 }
@@ -489,7 +489,8 @@ async fn idempotent_file_parts_are_compressed_and_named_for_it() {
     drop(publisher);
 
     // One part file, named for the codec, holding one gzip member with both records.
-    let part = output.join("part-orders-0-0-1.jsonl.gz");
+    let part =
+        output.join("part-orders-0000000000-00000000000000000000-00000000000000000001.jsonl.gz");
     let raw = std::fs::read(&part).unwrap();
     let plain =
         crate::support::compression::decompress_all(crate::models::Compression::Gzip, &raw, None)
@@ -512,7 +513,12 @@ async fn idempotent_file_parts_are_compressed_and_named_for_it() {
         .unwrap()
         .map(|entry| entry.unwrap().file_name().into_string().unwrap())
         .collect::<Vec<_>>();
-    assert_eq!(names, vec!["part-orders-0-0-1.jsonl.gz".to_string()]);
+    assert_eq!(
+        names,
+        vec![
+            "part-orders-0000000000-00000000000000000000-00000000000000000001.jsonl.gz".to_string()
+        ]
+    );
 }
 
 #[tokio::test]
@@ -533,6 +539,157 @@ async fn idempotent_file_sink_rejects_records_without_source_metadata() {
     assert!(std::fs::read_dir(output).unwrap().next().is_none());
 }
 
+#[tokio::test]
+async fn file_source_metadata_numbers_records_and_feeds_an_idempotent_sink() {
+    use crate::traits::MessageConsumer;
+
+    let dir = tempdir().unwrap();
+    let input = dir.path().join("orders.jsonl");
+    std::fs::write(&input, "{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n").unwrap();
+
+    let source = FileConfig {
+        path: input.to_string_lossy().into_owned(),
+        source_metadata: true,
+        ..Default::default()
+    };
+    let mut consumer = FileConsumer::new(&source).await.unwrap();
+    let batch = consumer.receive_batch(10).await.unwrap();
+    assert_eq!(batch.messages.len(), 3);
+
+    // Records are numbered by index, not byte offset, so they stay consecutive.
+    let records = batch
+        .messages
+        .iter()
+        .map(|m| m.metadata.get("mqb.src.file_record").unwrap().as_str())
+        .collect::<Vec<_>>();
+    assert_eq!(records, vec!["0", "1", "2"]);
+
+    // The whole batch lands as one part file covering records 0-2.
+    let output = dir.path().join("parts");
+    let sink = FileConfig {
+        path: output.to_string_lossy().into_owned(),
+        idempotency: true,
+        ..Default::default()
+    };
+    let publisher = FilePublisher::new(&sink).await.unwrap();
+    publisher.send_batch(batch.messages).await.unwrap();
+
+    let names = std::fs::read_dir(&output)
+        .unwrap()
+        .map(|entry| entry.unwrap().file_name().into_string().unwrap())
+        .filter(|name| !name.starts_with(".stage"))
+        .collect::<Vec<_>>();
+    assert_eq!(names.len(), 1, "one object per contiguous run: {names:?}");
+    assert!(
+        names[0].ends_with("-00000000000000000000-00000000000000000002.jsonl"),
+        "unexpected part name {}",
+        names[0]
+    );
+}
+
+#[tokio::test]
+async fn resuming_file_modes_get_a_run_epoch_so_reruns_cannot_reuse_a_record_index() {
+    use crate::support::source_ranges::SourcePosition;
+    use crate::traits::MessageConsumer;
+
+    let dir = tempdir().unwrap();
+    let input = dir.path().join("orders.jsonl");
+    std::fs::write(&input, "{\"id\":1}\n{\"id\":2}\n").unwrap();
+
+    let config = FileConfig {
+        path: input.to_string_lossy().into_owned(),
+        source_metadata: true,
+        mode: Some(FileConsumerMode::GroupSubscribe {
+            group_id: "g1".into(),
+            read_from_tail: false,
+        }),
+        ..Default::default()
+    };
+
+    // Allowed, not rejected: the setup is only weaker, not wrong.
+    let mut first = FileConsumer::new(&config).await.unwrap();
+    let batch = first.receive_batch(10).await.unwrap();
+    assert!(!batch.messages.is_empty());
+    let first_run = SourcePosition::from_message(&batch.messages[0]).unwrap();
+    assert!(batch.messages[0]
+        .metadata
+        .contains_key("mqb.src.file_epoch"));
+
+    // A second run restarts the record index at 0, so the epoch is what stops it from
+    // naming those records the same as the first run's and having them dropped.
+    let mut second = FileConsumer::new(&config).await.unwrap();
+    let batch = second.receive_batch(10).await.unwrap();
+    let second_run = SourcePosition::from_message(&batch.messages[0]).unwrap();
+
+    assert_ne!(first_run.source, second_run.source);
+    // A later run reads later records, so its objects must sort after the earlier run's.
+    assert!(first_run.source < second_run.source);
+}
+
+#[tokio::test]
+async fn run_epochs_are_distinct_for_consumers_created_in_the_same_millisecond() {
+    use crate::support::source_ranges::SourcePosition;
+
+    let dir = tempdir().unwrap();
+    let input = dir.path().join("orders.jsonl");
+    std::fs::write(&input, "{\"id\":1}\n").unwrap();
+
+    let config = FileConfig {
+        path: input.to_string_lossy().into_owned(),
+        source_metadata: true,
+        mode: Some(FileConsumerMode::GroupSubscribe {
+            group_id: "same-ms".into(),
+            read_from_tail: false,
+        }),
+        ..Default::default()
+    };
+
+    // No sleep between them: the epoch is allocated monotonically, not read off the clock.
+    let mut first = FileConsumer::new(&config).await.unwrap();
+    let mut second = FileConsumer::new(&config).await.unwrap();
+
+    let a = first.receive_batch(10).await.unwrap();
+    let b = second.receive_batch(10).await.unwrap();
+    let first_run = SourcePosition::from_message(&a.messages[0]).unwrap();
+    let second_run = SourcePosition::from_message(&b.messages[0]).unwrap();
+
+    assert!(
+        first_run.source < second_run.source,
+        "epochs must be distinct and increasing: {:?} vs {:?}",
+        first_run.source,
+        second_run.source
+    );
+}
+
+#[tokio::test]
+async fn consume_mode_repeats_its_record_identity_across_runs() {
+    use crate::support::source_ranges::SourcePosition;
+    use crate::traits::MessageConsumer;
+
+    let dir = tempdir().unwrap();
+    let input = dir.path().join("orders.jsonl");
+    std::fs::write(&input, "{\"id\":1}\n{\"id\":2}\n").unwrap();
+
+    let config = FileConfig {
+        path: input.to_string_lossy().into_owned(),
+        source_metadata: true,
+        ..Default::default()
+    };
+
+    let mut first = FileConsumer::new(&config).await.unwrap();
+    let a = first.receive_batch(10).await.unwrap();
+    let mut second = FileConsumer::new(&config).await.unwrap();
+    let b = second.receive_batch(10).await.unwrap();
+
+    // No epoch: re-reading the same file must produce the same names, which is exactly
+    // how the idempotent sink recognises the rewrite and skips it.
+    assert!(!a.messages[0].metadata.contains_key("mqb.src.file_epoch"));
+    assert_eq!(
+        SourcePosition::from_message(&a.messages[0]).unwrap(),
+        SourcePosition::from_message(&b.messages[0]).unwrap()
+    );
+}
+
 #[tokio::test]
 async fn idempotent_file_sink_replays_postgres_cdc_changes_in_one_commit() {
     fn postgres_message(ordinal: u64) -> crate::CanonicalMessage {
@@ -578,8 +735,8 @@ async fn idempotent_file_sink_replays_postgres_cdc_changes_in_one_commit() {
     assert_eq!(
         names,
         vec![
-            "part-postgres_cdc-bridge_slot-9876543210-0-0-1.jsonl".to_string(),
-            "part-postgres_cdc-bridge_slot-9876543210-0-2-2.jsonl".to_string(),
+            "part-postgres_cdc-bridge_slot-00000000009876543210-0000000000-00000000000000000000-00000000000000000001.jsonl".to_string(),
+            "part-postgres_cdc-bridge_slot-00000000009876543210-0000000000-00000000000000000002-00000000000000000002.jsonl".to_string(),
         ]
     );
 }

{mq_bridge_py_basic-0.4.2 → mq_bridge_py_basic-0.4.4}/src/endpoints/http/mod.rs

@@ -643,8 +643,9 @@ fn http_version_str(version: hyper::Version) -> &'static str {
     }
 }
 
-/// Credential-bearing request headers that must not be copied into message
-/// metadata, where they could be logged, persisted, or echoed back on the reply.
+/// Credential-bearing request headers. They reach metadata under
+/// [`SENSITIVE_HEADER_METADATA_PREFIX`] rather than under their own name, so they stop at the
+/// route handler instead of being forwarded.
 fn is_sensitive_request_header(name: &str) -> bool {
     const SENSITIVE: [&str; 6] = [
         "authorization",
@@ -657,6 +658,15 @@ fn is_sensitive_request_header(name: &str) -> bool {
     SENSITIVE.iter().any(|h| name.eq_ignore_ascii_case(h))
 }
 
+/// Metadata namespace for credential-bearing request headers, e.g. `mqb.src.http_authorization`,
+/// keeping the header name verbatim after the prefix.
+///
+/// Sits under [`SOURCE_METADATA_PREFIX`](crate::canonical_message::SOURCE_METADATA_PREFIX), which
+/// is what makes carrying the value safe: every publisher drops this prefix when serializing
+/// metadata, so the route handler sees the header the client actually sent while no sink, and no
+/// response header, ever does. See the `sensitive_header_*` tests.
+pub const SENSITIVE_HEADER_METADATA_PREFIX: &str = "mqb.src.http_";
+
 fn request_metadata_matches(request: &RequestMetadataView<'_>, key: &str, value: &str) -> bool {
     match key {
         HTTP_METHOD => request.method.as_str() == value,
@@ -1586,16 +1596,25 @@ async fn handle_request_internal(
                 content_encoding = Some(v_str.to_string());
             }
             let k_str = key.as_str();
+            // `mqb.src.*` is reserved for framework provenance, so a request header may never
+            // supply one: without this a client could forge its own `mqb.src.http_authorization`.
             if k_str == HTTP_METHOD
                 || k_str == HTTP_PATH
                 || k_str == HTTP_QUERY
                 || k_str == HTTP_VERSION
                 || k_str.eq_ignore_ascii_case("tls_cipher_suite")
                 || k_str.eq_ignore_ascii_case("tls_protocol_version")
-                || is_sensitive_request_header(k_str)
+                || crate::canonical_message::is_source_metadata_key(k_str)
             {
                 continue;
             }
+            if is_sensitive_request_header(k_str) {
+                metadata.insert(
+                    format!("{SENSITIVE_HEADER_METADATA_PREFIX}{k_str}"),
+                    v_str.to_string(),
+                );
+                continue;
+            }
             metadata.insert(k_str.to_string(), v_str.to_string());
         }
     }