apache-iggy 0.8.1.dev4__tar.gz → 0.9.0.dev1__tar.gz

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/Cargo.toml

@@ -200,13 +200,14 @@ hyper-util = { version = "0.1.20", features = ["server-auto", "service"] }
 iceberg = "0.9.1"
 iceberg-catalog-rest = "0.9.1"
 iceberg-storage-opendal = "0.9.1"
-iggy = { path = "core/sdk", version = "0.10.3-edge.3" }
-iggy-cli = { path = "core/cli", version = "0.13.1-edge.1" }
-iggy_binary_protocol = { path = "core/binary_protocol", version = "0.10.3-edge.3" }
-iggy_common = { path = "core/common", version = "0.10.3-edge.3" }
+iggy = { path = "core/sdk", version = "0.11.0-edge.1" }
+iggy-cli = { path = "core/cli", version = "0.14.0-edge.1" }
+iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.0-edge.1" }
+iggy_common = { path = "core/common", version = "0.11.0-edge.1" }
 iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.3.1-edge.1" }
 indexmap = "2.14.0"
 integration = { path = "core/integration" }
+ipnet = "2.12.0"
 journal = { path = "core/journal" }
 js-sys = "0.3"
 jsonwebtoken = { version = "10.4.0", features = ["rust_crypto"] }

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-iggy
-Version: 0.8.1.dev4
+Version: 0.9.0.dev1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/apache_iggy.pyi

@@ -44,6 +44,8 @@ __all__ = [
     "PollingStrategy",
     "ReceiveMessage",
     "SendMessage",
+    "SendMessagesConfirmation",
+    "SendMessagesResponse",
     "StreamDetails",
     "StreamPermissions",
     "Topic",
@@ -1212,10 +1214,13 @@ class IggyClient:
         topic: builtins.str | builtins.int,
         partitioning: builtins.int,
         messages: list[SendMessage],
-    ) -> collections.abc.Awaitable[None]:
+    ) -> collections.abc.Awaitable[SendMessagesResponse]:
         r"""
         Sends a list of messages to the specified topic.
-        Returns Ok(()) on successful sending or a PyRuntimeError on failure.
+        Returns a SendMessagesResponse carrying the per-partition commit
+        confirmations, or a PyRuntimeError on failure. The confirmation list is
+        empty when the server reports no offsets, and the legacy server never
+        reports any.
         """
     def poll_messages(
         self,
@@ -1606,6 +1611,65 @@ class SendMessage:
         directly from Python using the provided string or bytes data.
         """
 
+@typing.final
+class SendMessagesConfirmation:
+    r"""
+    A Python class representing the commit confirmation for one partition
+    written by a send.
+    """
+    @property
+    def stream_id(self) -> builtins.int:
+        r"""
+        Gets the unique identifier (numeric) of the stream the batch was written to.
+        """
+    @property
+    def topic_id(self) -> builtins.int:
+        r"""
+        Gets the unique identifier (numeric) of the topic the batch was written to.
+        """
+    @property
+    def partition_id(self) -> builtins.int:
+        r"""
+        Gets the identifier of the partition the batch was written to.
+        """
+    @property
+    def base_offset(self) -> builtins.int:
+        r"""
+        Gets the offset assigned to the first message of the batch in this partition.
+
+        The offset locates the batch, it does not identify it. Delivery is
+        at-least-once, so an earlier retry may already have committed these
+        messages at a lower offset.
+
+        A batch is confirmed once it is committed in memory, not once it is
+        fsynced. A crash-restart can stamp a later batch with an offset a client
+        has already recorded.
+
+        The legacy server confirms nothing, so its confirmation list is empty
+        and this value is never reached.
+        """
+
+@typing.final
+class SendMessagesResponse:
+    r"""
+    A Python class representing the outcome of a successful send.
+    """
+    @property
+    def confirmations(self) -> builtins.list[SendMessagesConfirmation]:
+        r"""
+        Gets the commit confirmations, one per partition the batch was written to.
+
+        The list is empty when the server reports no offsets, and the legacy
+        server never reports any, so branch on it being empty rather than
+        indexing into it.
+
+        A reported `base_offset` never implies uniqueness, because delivery is
+        at-least-once and an earlier retry may already have committed the same
+        messages at a lower offset. A batch is confirmed once it is committed in
+        memory, not once it is fsynced. A crash-restart can stamp a later batch
+        with an offset a client has already recorded.
+        """
+
 @typing.final
 class StreamDetails:
     @property

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/binary_protocol/Cargo.toml

@@ -17,7 +17,7 @@
 
 [package]
 name = "iggy_binary_protocol"
-version = "0.10.3-edge.3"
+version = "0.11.0-edge.1"
 description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK."
 edition = "2024"
 rust-version.workspace = true

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/binary_protocol/src/responses/messages/mod.rs

@@ -16,5 +16,7 @@
 // under the License.
 
 pub mod poll_messages;
+pub mod send_messages;
 
 pub use poll_messages::{PollMessagesResponse, PollMessagesResponseHeader};
+pub use send_messages::{SendMessagesConfirmationResponse, SendMessagesResponse};

apache_iggy-0.9.0.dev1/core/binary_protocol/src/responses/messages/send_messages.rs

@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le};
+use crate::error::WireError;
+use bytes::{BufMut, BytesMut};
+use std::borrow::Cow;
+
+/// Size of one confirmation entry:
+/// `stream_id(4) + topic_id(4) + partition_id(4) + base_offset(8)`.
+///
+/// Frozen. Entries are read at this stride out of an array whose length the
+/// peer chooses, so widening one is not a compatible change: a reader built
+/// against the old stride would take its second entry from the middle of the
+/// first and return the garbage as a successful decode. Extra data needs a new
+/// response type or a protocol-version gate. The entry count may grow freely.
+const CONFIRMATION_SIZE: usize = 20;
+
+/// Commit confirmation for one partition written by a `SendMessages` request.
+///
+/// Wire format:
+/// ```text
+/// [stream_id:4][topic_id:4][partition_id:4][base_offset:8]
+/// ```
+///
+/// `base_offset` is the offset assigned to the first message of the batch in
+/// that partition, bounded by three properties of the send path:
+/// - Delivery is at-least-once. An earlier retry of the same batch may already
+///   have committed at a lower offset, so the value never implies uniqueness.
+/// - A batch is confirmed once it is committed in memory, not once it is
+///   fsynced. A crash-restart can stamp a later batch with an offset a client
+///   has already recorded.
+/// - The legacy server confirms nothing, so its confirmation list is empty.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SendMessagesConfirmationResponse {
+    pub stream_id: u32,
+    pub topic_id: u32,
+    pub partition_id: u32,
+    pub base_offset: u64,
+}
+
+impl WireEncode for SendMessagesConfirmationResponse {
+    fn encoded_size(&self) -> usize {
+        CONFIRMATION_SIZE
+    }
+
+    fn encode(&self, buf: &mut BytesMut) {
+        buf.put_u32_le(self.stream_id);
+        buf.put_u32_le(self.topic_id);
+        buf.put_u32_le(self.partition_id);
+        buf.put_u64_le(self.base_offset);
+    }
+}
+
+impl WireDecode for SendMessagesConfirmationResponse {
+    fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+        let stream_id = read_u32_le(buf, 0)?;
+        let topic_id = read_u32_le(buf, 4)?;
+        let partition_id = read_u32_le(buf, 8)?;
+        let base_offset = read_u64_le(buf, 12)?;
+        Ok((
+            Self {
+                stream_id,
+                topic_id,
+                partition_id,
+                base_offset,
+            },
+            CONFIRMATION_SIZE,
+        ))
+    }
+}
+
+/// `SendMessages` response.
+///
+/// Wire format:
+/// ```text
+/// [confirmations_count:4][SendMessagesConfirmationResponse]*
+/// ```
+///
+/// `confirmations_count == 0` means the batch committed with no offsets to
+/// report. The legacy server goes further and answers a successful send with no
+/// body at all, so a caller sees an empty list either way and must handle it.
+///
+/// The server currently reports a single partition per request; the list
+/// decodes any count, so a later multi-partition send needs no wire change.
+/// Growing the count is the only compatible way to extend this response, the
+/// entry itself being frozen at 20 bytes.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SendMessagesResponse {
+    pub confirmations: Vec<SendMessagesConfirmationResponse>,
+}
+
+impl WireEncode for SendMessagesResponse {
+    fn encoded_size(&self) -> usize {
+        4 + self.confirmations.len() * CONFIRMATION_SIZE
+    }
+
+    #[allow(clippy::cast_possible_truncation)]
+    fn encode(&self, buf: &mut BytesMut) {
+        buf.put_u32_le(self.confirmations.len() as u32);
+        for confirmation in &self.confirmations {
+            confirmation.encode(buf);
+        }
+    }
+}
+
+impl WireDecode for SendMessagesResponse {
+    fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> {
+        let confirmations_count = read_u32_le(buf, 0)?;
+        let remaining = buf.len().saturating_sub(4);
+        let mut confirmations = Vec::with_capacity(capped_capacity(
+            confirmations_count as usize,
+            remaining,
+            CONFIRMATION_SIZE,
+        ));
+        let mut offset = 4;
+        for _ in 0..confirmations_count {
+            let (confirmation, consumed) =
+                SendMessagesConfirmationResponse::decode(&buf[offset..])?;
+            offset += consumed;
+            confirmations.push(confirmation);
+        }
+        // The payload is the whole frame body, never a prefix of a larger
+        // value, so leftover bytes mean a shape this build cannot read.
+        // Tolerating a tail would be sound only where the extra bytes cannot
+        // precede a field the reader still indexes, which a fixed-stride array
+        // breaks at any count above one: entries past the first would be read
+        // from inside their predecessor and returned as a successful decode.
+        if offset != buf.len() {
+            return Err(WireError::Validation(Cow::Owned(format!(
+                "send_messages response has {} trailing bytes",
+                buf.len() - offset
+            ))));
+        }
+        Ok((Self { confirmations }, offset))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn confirmation(partition_id: u32) -> SendMessagesConfirmationResponse {
+        SendMessagesConfirmationResponse {
+            stream_id: 1,
+            topic_id: 2,
+            partition_id,
+            base_offset: 42,
+        }
+    }
+
+    #[test]
+    fn roundtrip_single_confirmation() {
+        let response = SendMessagesResponse {
+            confirmations: vec![confirmation(3)],
+        };
+        let bytes = response.to_bytes();
+        assert_eq!(bytes.len(), 4 + CONFIRMATION_SIZE);
+        let (decoded, consumed) = SendMessagesResponse::decode(&bytes).unwrap();
+        assert_eq!(consumed, bytes.len());
+        assert_eq!(decoded, response);
+    }
+
+    #[test]
+    fn roundtrip_multiple_confirmations() {
+        let response = SendMessagesResponse {
+            confirmations: vec![confirmation(0), confirmation(1), confirmation(2)],
+        };
+        let bytes = response.to_bytes();
+        let (decoded, consumed) = SendMessagesResponse::decode(&bytes).unwrap();
+        assert_eq!(consumed, bytes.len());
+        assert_eq!(decoded, response);
+    }
+
+    #[test]
+    fn roundtrip_zero_confirmations() {
+        let response = SendMessagesResponse {
+            confirmations: vec![],
+        };
+        let bytes = response.to_bytes();
+        assert_eq!(&bytes[..], &[0x00, 0x00, 0x00, 0x00]);
+        let (decoded, consumed) = SendMessagesResponse::decode(&bytes).unwrap();
+        assert_eq!(consumed, 4);
+        assert_eq!(decoded, response);
+    }
+
+    #[test]
+    fn wire_compat_confirmation_layout() {
+        let response = SendMessagesResponse {
+            confirmations: vec![SendMessagesConfirmationResponse {
+                stream_id: 1,
+                topic_id: 2,
+                partition_id: 3,
+                base_offset: 4,
+            }],
+        };
+        assert_eq!(
+            &response.to_bytes()[..],
+            &[
+                0x01, 0x00, 0x00, 0x00, // count
+                0x01, 0x00, 0x00, 0x00, // stream_id
+                0x02, 0x00, 0x00, 0x00, // topic_id
+                0x03, 0x00, 0x00, 0x00, // partition_id
+                0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // base_offset
+            ]
+        );
+    }
+
+    #[test]
+    fn confirmation_truncated_returns_error() {
+        let bytes = confirmation(1).to_bytes();
+        for i in 0..bytes.len() {
+            assert!(
+                SendMessagesConfirmationResponse::decode(&bytes[..i]).is_err(),
+                "expected error for truncation at byte {i}"
+            );
+        }
+    }
+
+    #[test]
+    fn truncated_returns_error() {
+        let response = SendMessagesResponse {
+            confirmations: vec![confirmation(0), confirmation(1)],
+        };
+        let bytes = response.to_bytes();
+        for i in 0..bytes.len() {
+            assert!(
+                SendMessagesResponse::decode(&bytes[..i]).is_err(),
+                "expected error for truncation at byte {i}"
+            );
+        }
+    }
+
+    #[test]
+    fn trailing_bytes_rejected() {
+        let response = SendMessagesResponse {
+            confirmations: vec![confirmation(1)],
+        };
+        let mut bytes = response.to_bytes().to_vec();
+        bytes.push(0xFF);
+        assert!(matches!(
+            SendMessagesResponse::decode(&bytes),
+            Err(WireError::Validation(_))
+        ));
+    }
+
+    #[test]
+    fn trailing_bytes_after_zero_confirmations_rejected() {
+        let bytes = [0x00, 0x00, 0x00, 0x00, 0x00];
+        assert!(matches!(
+            SendMessagesResponse::decode(&bytes),
+            Err(WireError::Validation(_))
+        ));
+    }
+
+    #[test]
+    fn empty_input_returns_error() {
+        assert!(matches!(
+            SendMessagesResponse::decode(&[]),
+            Err(WireError::UnexpectedEof { .. })
+        ));
+    }
+
+    #[test]
+    fn bogus_confirmations_count_does_not_oom() {
+        let mut buf = BytesMut::new();
+        buf.put_u32_le(u32::MAX);
+        assert!(SendMessagesResponse::decode(&buf).is_err());
+    }
+}

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/common/Cargo.toml

@@ -17,7 +17,7 @@
 
 [package]
 name = "iggy_common"
-version = "0.10.3-edge.3"
+version = "0.11.0-edge.1"
 description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second."
 edition = "2024"
 rust-version.workspace = true

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/common/src/error/iggy_error.rs

@@ -17,7 +17,7 @@
 
 use crate::Identifier;
 use crate::utils::topic_size::MaxTopicSize;
-use crate::{IggyMessage, utils::byte_size::IggyByteSize};
+use crate::{IggyMessage, SendMessagesConfirmationResponse, utils::byte_size::IggyByteSize};
 use std::sync::Arc;
 use strum::{EnumDiscriminants, FromRepr, IntoStaticStr};
 use thiserror::Error;
@@ -416,6 +416,9 @@ pub enum IggyError {
     ProducerSendFailed {
         cause: Box<IggyError>,
         failed: Arc<Vec<IggyMessage>>,
+        /// Confirmations of the chunks that committed before `cause`; the
+        /// durable prefix of a send that was split into several requests.
+        committed: Arc<Vec<SendMessagesConfirmationResponse>>,
         stream_name: String,
         topic_name: String,
     } = 4056,

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/common/src/http/messages/send_messages.rs

@@ -22,7 +22,9 @@ use crate::Validatable;
 use crate::error::IggyError;
 use crate::types::message::HeaderEntry;
 use crate::types::message::partitioning::Partitioning;
-use crate::{IggyMessage, IggyMessagesBatch};
+use crate::{
+    IggyMessage, IggyMessagesBatch, SendMessagesConfirmationResponse, SendMessagesResponse,
+};
 use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
 use bytes::Bytes;
 use serde::de::{self, MapAccess, Visitor};
@@ -302,6 +304,60 @@ impl<'de> Deserialize<'de> for SendMessages {
     }
 }
 
+/// JSON body of a successful `POST .../messages`: one entry per partition the
+/// batch landed in. A list rather than a single confirmation, so a
+/// multi-partition produce needs no shape change.
+///
+/// Distinct from the binary `SendMessagesResponse` because wire types stay
+/// codec-only and carry no `serde` derives.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SendMessagesConfirmations {
+    pub confirmations: Vec<SendMessagesConfirmation>,
+}
+
+/// One partition's commit confirmation.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SendMessagesConfirmation {
+    pub stream_id: u32,
+    pub topic_id: u32,
+    pub partition_id: u32,
+    pub base_offset: u64,
+}
+
+impl From<SendMessagesResponse> for SendMessagesConfirmations {
+    fn from(response: SendMessagesResponse) -> Self {
+        Self {
+            confirmations: response
+                .confirmations
+                .into_iter()
+                .map(|confirmation| SendMessagesConfirmation {
+                    stream_id: confirmation.stream_id,
+                    topic_id: confirmation.topic_id,
+                    partition_id: confirmation.partition_id,
+                    base_offset: confirmation.base_offset,
+                })
+                .collect(),
+        }
+    }
+}
+
+impl From<SendMessagesConfirmations> for SendMessagesResponse {
+    fn from(body: SendMessagesConfirmations) -> Self {
+        Self {
+            confirmations: body
+                .confirmations
+                .into_iter()
+                .map(|confirmation| SendMessagesConfirmationResponse {
+                    stream_id: confirmation.stream_id,
+                    topic_id: confirmation.topic_id,
+                    partition_id: confirmation.partition_id,
+                    base_offset: confirmation.base_offset,
+                })
+                .collect(),
+        }
+    }
+}
+
 fn parse_message_id(value: Option<&serde_json::Value>) -> Result<u128, String> {
     let value = match value {
         Some(v) => v,

{apache_iggy-0.8.1.dev4 → apache_iggy-0.9.0.dev1}/core/common/src/lib.rs

@@ -52,6 +52,9 @@ pub use http::streams::*;
 pub use http::system::*;
 pub use http::topics::*;
 pub use http::users::*;
+pub use iggy_binary_protocol::responses::messages::{
+    SendMessagesConfirmationResponse, SendMessagesResponse,
+};
 pub use traits::binary_client::BinaryClient;
 pub use traits::binary_transport::BinaryTransport;
 #[cfg(feature = "vsr")]
@@ -60,6 +63,7 @@ pub use traits::client::Client;
 pub use traits::cluster_client::ClusterClient;
 pub use traits::consumer_group_client::ConsumerGroupClient;
 pub use traits::consumer_offset_client::ConsumerOffsetClient;
+pub use traits::decode_send_confirmations;
 pub use traits::message_client::MessageClient;
 pub use traits::partition_client::PartitionClient;
 pub use traits::partitioner::Partitioner;