mq-bridge-py-basic 0.4.1__tar.gz → 0.4.3__tar.gz

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/Cargo.lock

@@ -1900,9 +1900,9 @@ dependencies = [
 
 [[package]]
 name = "fast-uuid-v7"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65d7a30239eba18ae6a60dc3c3df98af18f00bba69d2b1d5f2816a831e83bff3"
+checksum = "dc759201eb9b09178fb2174087a451d19fd46a8b8b4f5f6103de83b9f5949f52"
 dependencies = [
  "rand 0.10.2",
 ]
@@ -3348,7 +3348,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -3431,7 +3431,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-bindings-common"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "anyhow",
  "fast-uuid-v7",
@@ -3445,7 +3445,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-node"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -3487,7 +3487,7 @@ dependencies = [
 
 [[package]]
 name = "mq-bridge-py"
-version = "0.4.1"
+version = "0.4.3"
 dependencies = [
  "anyhow",
  "async-trait",

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/Cargo.toml

@@ -16,13 +16,13 @@ members = ["bindings-common", "python/mq-bridge-py"]
 resolver = "2"
 
 [workspace.package]
-version = "0.4.1"
+version = "0.4.3"
 
 [workspace.dependencies]
 anyhow = "1.0"
 async-trait = "0.1"
 bytes = "1"
-fast-uuid-v7 = "0.1.8"
+fast-uuid-v7 = "0.1.9"
 serde = "1.0"
 serde_json = { version = "1.0", features = ["raw_value"] }
 serde_yaml_ng = "0.10"

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mq-bridge-py-basic
-Version: 0.4.1
+Version: 0.4.3
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/docs/ARCHITECTURE.md

@@ -150,6 +150,19 @@ Batch processing is a core concept in mq-bridge and is required for all endpoint
 - Each route can be configured with a `concurrency` parameter, which determines how many worker tasks will process batches in parallel.
 - Batch size is also configurable per route.
 
+#### Ordering
+Two independent guarantees, each decided once per route by the endpoint itself:
+
+- **Commits** follow `MessageConsumer::commit_requires_order()` (default `true`). Cumulative-ack sources such as Kafka funnel their commits through one sequencer; individually-acking sources commit concurrently, bounded by `commit_concurrency_limit`.
+- **Publishing** follows `MessagePublisher::requires_ordered_publish()` (default `false`). Above `concurrency: 1` workers call `send_batch` in parallel, so whole batches can reach the sink out of source order — rows keep their order *within* a batch. The `file` sink declares `true` and the route sequences the `send_batch` calls; batch prep and commits stay parallel, so the ordered path measures within noise of an unordered one.
+
+Only `file` declares it, for two reasons that any other candidate has to clear:
+
+- **The cost must be low.** The file sink already holds a write lock across the whole batch, so sequencing changes who writes, not how many write at once. Sequencing a sink that ends in a network round trip instead costs the whole concurrency factor — one batch in flight instead of N.
+- **The guarantee must be reachable.** `object_store` does *not* declare it: read order there is object-key order, and the uuidv7 key randomises everything below the millisecond, so ordering the writes would buy nothing. Ordered cloud export needs a source-sequenced key first.
+
+Broker sinks keep it off. NATS (both Core and JetStream) publishes a batch through `send_batch_helper`, which keeps up to `SEND_BATCH_CONCURRENCY` publishes in flight, so wire order inside one batch is already unordered — only the results are re-sorted. Kafka and AMQP preserve order within a batch (they submit the whole batch, then await confirms), but for them the order-critical part is the cheap submit loop, not the round trip, so a useful fix would have to release the sequence after submit rather than after `send_batch`. For per-key Kafka ordering today, use `concurrency: 1`.
+
 ### Helper Utilities
 mq-bridge provides several helper functions to make implementing batching easier, especially if your endpoint only supports single-message operations:
 

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/docs/DELIVERY.md

@@ -22,7 +22,7 @@ sink writes data.
 | MongoDB | `id_field` set to a payload field or replay-stable template such as `${metadata:mqb.id}` |
 | PostgreSQL / SQLite | `sqlx.insert_query` uses a unique key with `ON CONFLICT` |
 | MySQL / MariaDB | `sqlx.insert_query` uses a unique key with `ON DUPLICATE KEY` |
-| File / object store | `idempotency: true` with a replayable Kafka or Postgres CDC source |
+| File / object store | `idempotency: true` with a replayable Kafka, Postgres CDC, SQL cursor, MongoDB CDC or file source |
 
 For a source without stable identity, derive one once and consume it at the sink:
 
@@ -143,7 +143,7 @@ one identity to every message missing the field, so it is dropped instead.
 | `mongodb` | Unique `_id` index; dup-key (11000) is treated as an idempotent success | `id_field` |
 | `sqlx` (PostgreSQL / MySQL / SQLite) | The table's own `UNIQUE`/`PRIMARY KEY` | `ON CONFLICT` / `ON DUPLICATE KEY` in `insert_query` |
 | `clickhouse` | `ReplacingMergeTree` collapses at merge time | Table DDL — no `mq-bridge` config |
-| `file`, `object_store` | Deterministic part names + covered-range recovery | `idempotency: true` (needs a replayable source) |
+| `file`, `object_store` | Deterministic, sortable part names + covered-range recovery | `idempotency: true` (needs a replayable source) |
 | `kafka` | `enable.idempotence` dedups **producer retries within one session** — this is *not* exactly-once semantics | On by default |
 | `nats`, `amqp`, `mqtt`, `redis_streams`, `aws`, `ibm_mq`, `zeromq` | None | Deduplicate at the next consumer instead |
 
@@ -359,6 +359,13 @@ but before the branch's downstream send committed, the replay hits a duplicate k
 
 ### Files & object storage — `idempotency`
 
+> **Check first whether you need any of this.** If your records already carry a business key — an
+> `id` field in the payload, or `mqb.id` from the [`id` middleware](#giving-a-source-an-identity) —
+> then a key-addressed sink (Mongo `id_field`, SQL `ON CONFLICT`) already gives you effective
+> exactly-once, and the order objects happen to land in does not matter. Positional naming below is
+> for the case where the *sink itself* has to recognise a replay, or where a downstream reader
+> depends on replay **order**. Turning it on when you do not need it only adds restrictions.
+
 A filesystem has no unique constraint, so the `file` and `object_store` sinks get replay safety a
 different way: **deterministic names plus covered-range recovery**. Set `idempotency: true` and the
 sink stops writing UUID-named objects. Instead it groups each batch into runs of consecutive source
@@ -371,7 +378,7 @@ kafka_to_s3:
   output:
     object_store:
       url: "s3://my-bucket/orders"
-      idempotency: true          # → s3://my-bucket/orders/part-orders-0-0-1023.jsonl
+      idempotency: true          # → part-orders-<partition>-<start>-<end>.jsonl (zero-padded)
 ```
 
 On startup the sink lists what is already there, parses the ranges out of the part names, and drops
@@ -381,13 +388,58 @@ checkpoint protocol. Local files stage to a temp name, `fsync`, then `rename` (a
 filesystem); object stores have no atomic rename, so a single PUT under the final name *is* the
 commit.
 
-This needs a replayable source position, which today means **Kafka** (topic/partition/offset) or
-**`postgres_cdc`** (commit LSN plus in-transaction ordinal; `sqlx` with `publication` maps onto the
-same consumer). A route with an idempotent output turns `source_metadata` on for its input
-automatically; set it explicitly only when you want the `mqb.src.*` keys for something else. Any
-other input is rejected when the route starts. NATS, AMQP and MongoDB CDC also accept
-`source_metadata` and emit provenance keys, but a subject or routing key is not a replayable offset,
-so they cannot drive an idempotent sink.
+This needs a replayable source position. Today that means:
+
+| Source | Position | Restriction |
+| --- | --- | --- |
+| `kafka` | topic / partition / offset | — |
+| `postgres_cdc` | commit LSN + in-transaction ordinal | `sqlx` with `publication` maps onto the same consumer |
+| `mongodb` | cluster time + ordinal; the initial snapshot uses its `_id` scan index | `capture_new` / `capture_all` only — `consumer` and `snapshot` have no position |
+| `file` | record index within the file | all modes; only `consume` deduplicates across runs (see below) |
+| `sqlx` | the `cursor_column` value | cursor polling only; the column must be a unique, non-negative integer |
+
+A route with an idempotent output turns `source_metadata` on for its input automatically; set it
+explicitly only when you want the `mqb.src.*` keys for something else. Any other input is rejected
+when the route starts. NATS and AMQP also accept `source_metadata` and emit provenance keys, but a
+subject or routing key is not a replayable offset, so they cannot drive an idempotent sink.
+
+**Without `idempotency: true`, an `object_store` sink names objects in write order**, and key order
+is the only order a bucket has. At `concurrency: 1` that is source order; above it the name is minted
+inside the worker pool, so it is arrival order, and replaying a change stream through the bucket can
+reorder updates to the same key. The route logs a warning when it starts in that configuration.
+
+**Numbers inside a part name are zero-padded** so that ASCII sort equals numeric sort — a bucket is
+listed lexicographically and that listing order *is* the replay order. A bucket written by a version
+before this padding contains unpadded names; the two forms do not sort correctly against each other,
+so do not mix them in one prefix.
+
+**MongoDB `capture_all`** reads the collection, then streams changes. Both phases are numbered into
+the key so the snapshot always sorts ahead of the changes; a change can never replay before the
+document it modifies. The snapshot does not resume across runs — it re-scans from the start — but the
+numbering is deterministic, so a restart reproduces the same names and the covered-range recovery
+skips them. The exception is inserts or deletes landing in the collection between the two runs: those
+shift the numbering, and some documents are then written twice.
+
+**The `sqlx` cursor source** uses the `cursor_column` value itself as the position — the same shape
+Kafka Connect's S3 sink gives a Kafka offset. That only works for a unique, non-negative integer
+column: a text cursor pages fine but has no contiguous numeric order, and a repeated value would
+resolve two rows to one position and drop one of them. The reader rejects both at read time rather
+than naming records wrongly. Consecutive ids coalesce into one part file; gaps split it.
+
+**The `file` source** numbers records by their index in the file, not their byte offset (byte offsets
+are not consecutive, so every record would become its own part).
+
+That index repeats across runs only in `consume` mode, which always reads from byte 0 — and repeating
+is the point: a re-read produces the same names, so the sink recognises them as already covered and
+writes nothing. `subscribe` starts at the current end of the file and `group_subscribe` resumes at a
+stored byte offset, so their index restarts at 0 over records an earlier run already numbered. Those
+two modes therefore carry a **run epoch** in the object name.
+
+The epoch keeps names distinct and keeps runs in order (a later run reads later records, so its
+objects sort after the earlier run's). What it does not give you is deduplication across a restart:
+records re-read after a crash are written again under new names. That is ordinary at-least-once, and
+it is the honest guarantee for a source with no durable per-record position — these modes are
+allowed, not rejected, because that guarantee is fine for plenty of pipelines.
 
 For the `file` sink, `idempotency: true` changes what `path` means: it is the directory that receives
 the part files, not the file that is appended to. The sink creates it on startup, so pointing it at an

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/mq-bridge.schema.json

@@ -1012,13 +1012,18 @@
           "default": "normal"
         },
         "idempotency": {
-          "description": "Write replay-safe source ranges as immutable part files under this directory.\nRequires Kafka source metadata or postgres_cdc commit LSN plus transaction ordinal metadata.",
+          "description": "Write replay-safe source ranges as immutable part files under this directory.\nRequires a source that stamps a replayable position: kafka, postgres_cdc, mongodb CDC, sqlx or file.",
           "type": "boolean",
           "default": false
         },
         "path": {
           "description": "Path to the file, or to the directory holding the part files when `idempotency` is true.",
           "type": "string"
+        },
+        "source_metadata": {
+          "description": "(Consumer only) Include authoritative `mqb.src.file_*` source positions; only `consume` mode reproduces them across restarts. Defaults to false.",
+          "type": "boolean",
+          "default": false
         }
       },
       "anyOf": [
@@ -2972,7 +2977,7 @@
           "minimum": 1
         },
         "concurrency": {
-          "description": "(Optional) Number of concurrent processing tasks for this route. While it improves throughput for high-latency\nhandlers, it adds synchronization overhead for ordered commits and may lead to out-of-order processing\nin the handler. Defaults to 1.",
+          "description": "(Optional) Number of concurrent processing tasks for this route. While it improves throughput for high-latency\nhandlers, it adds synchronization overhead for ordered commits and may lead to out-of-order processing\nin the handler. Above 1, whole batches may also reach the sink out of source order (rows keep their order\nwithin a batch) unless the sink declares itself order-sensitive, as `file` does.\nDefaults to 1.",
           "type": "integer",
           "format": "uint",
           "default": 1,
@@ -3217,6 +3222,11 @@
             "null"
           ]
         },
+        "source_metadata": {
+          "description": "(Consumer only) Include authoritative `mqb.src.sqlx_*` source positions; `cursor_column` must then be a unique integer. Defaults to false.",
+          "type": "boolean",
+          "default": false
+        },
         "table": {
           "description": "The table to interact with.",
           "type": "string"

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/mq_bridge/config.pyi

@@ -154,6 +154,7 @@ class FileConfig(TypedDict, total=False):
     format: FileFormat
     idempotency: bool
     path: Required[str]
+    source_metadata: bool
 
 
 class GrpcConfig(TypedDict, total=False):
@@ -476,6 +477,7 @@ class SqlxConfig(TypedDict, total=False):
     select_query: Optional[str]
     shared: Optional[bool]
     slot_name: Optional[str]
+    source_metadata: bool
     table: Required[str]
     test_before_acquire: Optional[bool]
     tls: TlsConfig

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/src/command_handler.rs

@@ -165,6 +165,10 @@ impl MessagePublisher for CommandPublisher {
         self.inner.status().await
     }
 
+    fn requires_ordered_publish(&self) -> bool {
+        self.inner.requires_ordered_publish()
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }

{mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.3}/src/endpoints/file/mod.rs

@@ -19,7 +19,7 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::io::Seek;
 use std::path::Path;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex as StdMutex, Weak};
 use std::time::{Duration, SystemTime};
 use tokio::fs::{self, File, OpenOptions};
@@ -1017,6 +1017,12 @@ impl MessagePublisher for FilePublisher {
         Ok(())
     }
 
+    /// A file is an ordered log by construction: appending batches in the order they
+    /// were read is the whole point of exporting to JSONL/CSV.
+    fn requires_ordered_publish(&self) -> bool {
+        true
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -2007,15 +2013,44 @@ pub struct FileConsumer {
     /// A live tail keeps waiting for the file to appear; a drain (`exit_on_empty`)
     /// cannot, so it fails with this instead of blocking forever.
     startup_open_error: Option<String>,
+    /// Stamp `mqb.src.file_*` so an idempotent sink can name records in source order.
+    source_metadata: bool,
+    /// Index of the next record in this file.
+    next_record: u64,
+    /// Set only for the modes that do not start at byte 0, so their record indexes cannot
+    /// collide with a previous run's. `None` for `consume`, whose names must repeat.
+    run_epoch: Option<u64>,
     exit_on_empty: bool,
 }
 
+/// Hands out a strictly increasing run epoch. Seeded from wall-clock millis so epochs still
+/// sort across process restarts, but never repeats or goes backwards within one process —
+/// consumers built in the same millisecond, or across a clock step back, stay distinct.
+fn next_run_epoch() -> u64 {
+    static LAST: AtomicU64 = AtomicU64::new(0);
+    let now = SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_millis() as u64)
+        .unwrap_or(0);
+    let mut last = LAST.load(Ordering::Relaxed);
+    loop {
+        let epoch = now.max(last + 1);
+        match LAST.compare_exchange_weak(last, epoch, Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => return epoch,
+            Err(observed) => last = observed,
+        }
+    }
+}
+
 impl FileConsumer {
     fn wrap(backend: ConsumerBackend) -> Self {
         Self {
             backend,
             path: String::new(),
             startup_open_error: None,
+            source_metadata: false,
+            next_record: 0,
+            run_epoch: None,
             exit_on_empty: false,
         }
     }
@@ -2025,6 +2060,17 @@ impl FileConsumer {
         let mut consumer = Self::new_backend(config).await?;
         consumer.path = config.path.clone();
         consumer.startup_open_error = startup_open_error;
+        consumer.source_metadata = config.source_metadata;
+        // `consume` always reads from byte 0, so its record index is reproducible and two
+        // runs deliberately produce the same names — that is what makes the sink idempotent.
+        // `subscribe` starts at the current end and `group_subscribe` resumes at a stored
+        // byte offset, so their index restarts at 0 over records a previous run already
+        // numbered. Without an epoch those names would collide and the sink would discard
+        // the new records as already-covered; with one they stay distinct and ordered, at
+        // the cost of cross-restart deduplication.
+        consumer.run_epoch = (config.source_metadata
+            && !matches!(&config.mode, None | Some(FileConsumerMode::Consume { .. })))
+        .then(next_run_epoch);
         Ok(consumer)
     }
 
@@ -2338,6 +2384,37 @@ impl MessageConsumer for FileConsumer {
     // a cumulative byte offset (the max acked `file_offset`), so out-of-order
     // commits could advance the offset past un-acked messages and lose them.
     async fn receive_batch(&mut self, max_messages: usize) -> Result<ReceivedBatch, ConsumerError> {
+        let mut batch = self.receive_batch_inner(max_messages).await?;
+        if self.source_metadata {
+            for message in &mut batch.messages {
+                message
+                    .metadata
+                    .insert("mqb.src.file_path".to_string(), self.path.clone());
+                message.metadata.insert(
+                    "mqb.src.file_record".to_string(),
+                    self.next_record.to_string(),
+                );
+                if let Some(epoch) = self.run_epoch {
+                    message
+                        .metadata
+                        .insert("mqb.src.file_epoch".to_string(), epoch.to_string());
+                }
+                self.next_record += 1;
+            }
+        }
+        Ok(batch)
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl FileConsumer {
+    async fn receive_batch_inner(
+        &mut self,
+        max_messages: usize,
+    ) -> Result<ReceivedBatch, ConsumerError> {
         // The path was unopenable at startup. A live tail waits for it (rotation,
         // a writer that hasn't created it yet), but a drain has nothing to wait
         // for, so re-probe and fail rather than block forever on a typo'd path.
@@ -2560,10 +2637,6 @@ impl MessageConsumer for FileConsumer {
             }
         }
     }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
 }
 
 /// Wraps a message body for the Json/Text file formats. Generic over the payload type