apache-iggy 0.7.3.dev1__tar.gz → 0.7.5.dev1__tar.gz

{apache_iggy-0.7.3.dev1 → apache_iggy-0.7.5.dev1}/Cargo.toml

@@ -45,6 +45,7 @@ members = [
     "core/consensus",
     "core/harness_derive",
     "core/integration",
+    "core/iobuf",
     "core/journal",
     "core/message_bus",
     "core/metadata",
@@ -65,12 +66,13 @@ actix-files = "0.6.10"
 actix-web = "4.13.0"
 aes-gcm = "0.10.3"
 ahash = { version = "0.8.12", features = ["serde"] }
+aligned-vec = "0.6.4"
 anyhow = "1.0.102"
 argon2 = "0.5.3"
 arrow = "57.3.0"
 arrow-array = "57.3.0"
 arrow-json = "57.3.0"
-assert_cmd = "2.1.2"
+assert_cmd = "2.2.0"
 async-broadcast = "0.7.2"
 async-channel = "2.5.0"
 async-dropper = { version = "0.3.1", features = ["tokio", "simple"] }
@@ -84,16 +86,16 @@ bench-dashboard-server = { path = "core/bench/dashboard/server" }
 bench-dashboard-shared = { path = "core/bench/dashboard/shared" }
 bench-report = { path = "core/bench/report" }
 bench-runner = { path = "core/bench/runner" }
-bit-set = "0.8.0"
+bit-set = "0.9.1"
 blake3 = "1.8.3"
-bon = "3.9.0"
+bon = "3.9.1"
 byte-unit = { version = "5.2.0", default-features = false, features = ["serde", "byte", "std"] }
 bytemuck = { version = "1.25", features = ["derive", "min_const_generics"] }
 bytes = "1.11.1"
 charming = "0.6.0"
 chrono = { version = "0.4.44", features = ["serde"] }
-clap = { version = "4.5.60", features = ["derive", "wrap_help"] }
-clap_complete = "4.5.66"
+clap = { version = "4.6.0", features = ["derive", "wrap_help"] }
+clap_complete = "4.6.0"
 clock = { path = "core/clock" }
 colored = "3.1.1"
 comfy-table = "7.2.2"
@@ -117,11 +119,11 @@ configs_derive = { path = "core/configs_derive", version = "0.1.0" }
 consensus = { path = "core/consensus" }
 console-subscriber = "0.5.0"
 crossbeam = "0.8.4"
-crossfire = "3.1.5"
+crossfire = "3.1.7"
 ctor = "0.6.3"
 ctrlc = { version = "3.5", features = ["termination"] }
 cucumber = "0.22"
-cyper = { version = "0.8.2", features = ["rustls"], default-features = false }
+cyper = { version = "0.8.3", features = ["rustls"], default-features = false }
 cyper-axum = { version = "0.8.0" }
 darling = "0.23"
 dashmap = "6.1.0"
@@ -133,12 +135,11 @@ dirs = "6.0.0"
 dlopen2 = "0.8.2"
 dotenvy = "0.15.7"
 elasticsearch = { version = "9.1.0-alpha.1", features = ["rustls-tls"], default-features = false }
-enum_dispatch = "0.3.13"
 enumset = "1.1"
 env_logger = "0.11.9"
 err_trail = { version = "0.11.0", features = ["tracing"] }
 error_set = "0.9.1"
-figlet-rs = "0.1.5"
+figlet-rs = "1.0.0"
 figment = { version = "0.10.19", features = ["toml", "env"] }
 file-operation = "0.8.19"
 flatbuffers = "25.12.19"
@@ -156,13 +157,14 @@ hostname = "0.4.2"
 human-repr = "1.1.0"
 humantime = "2.3.0"
 hwlocality = "1.0.0-alpha.11"
-iceberg = "0.8.0"
-iceberg-catalog-rest = "0.8.0"
-iggy = { path = "core/sdk", version = "0.9.3-edge.1" }
-iggy-cli = { path = "core/cli", version = "0.11.3-edge.1" }
-iggy_binary_protocol = { path = "core/binary_protocol", version = "0.9.3-edge.1" }
-iggy_common = { path = "core/common", version = "0.9.3-edge.1" }
-iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.2.1-edge.1" }
+iceberg = "0.9.0"
+iceberg-catalog-rest = "0.9.0"
+iceberg-storage-opendal = "0.9.0"
+iggy = { path = "core/sdk", version = "0.9.5-edge.1" }
+iggy-cli = { path = "core/cli", version = "0.12.1-edge.1" }
+iggy_binary_protocol = { path = "core/binary_protocol", version = "0.9.5-edge.1" }
+iggy_common = { path = "core/common", version = "0.9.5-edge.1" }
+iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.2.2-edge.1" }
 integration = { path = "core/integration" }
 journal = { path = "core/journal" }
 js-sys = "0.3"
@@ -173,22 +175,22 @@ left-right = "0.11"
 lending-iterator = "0.1.7"
 libc = "0.2.183"
 log = "0.4.29"
-lz4_flex = "0.12.0"
+lz4_flex = "0.13.0"
 message_bus = { path = "core/message_bus" }
 metadata = { path = "core/metadata" }
 mimalloc = "0.1"
 mime_guess = "2.0"
 mockall = "0.14.0"
 moka = { version = "0.12.14", features = ["future"] }
-mongodb = { version = "3.5.1", features = ["rustls-tls"] }
-nix = { version = "0.31.2", features = ["fs", "resource", "sched"] }
+mongodb = { version = "3.5.2", features = ["rustls-tls"] }
+nix = { version = "0.31.2", features = ["feature", "fs", "resource", "sched"] }
 nonzero_lit = "0.1.2"
 notify = "8.2.0"
 octocrab = "0.49.5"
-once_cell = "1.21.3"
+once_cell = "1.21.4"
 opentelemetry = { version = "0.31.0", features = ["trace", "logs"] }
 opentelemetry-appender-tracing = { version = "0.31.1", features = ["log"] }
-opentelemetry-otlp = { version = "0.31.0", features = [
+opentelemetry-otlp = { version = "0.31.1", features = [
     "logs",
     "trace",
     "grpc-tonic",
@@ -230,7 +232,7 @@ reqwest-retry = "0.9.1"
 reqwest-tracing = "0.7.0"
 ring = "0.17.14"
 ringbuffer = "0.16.0"
-rmcp = "1.1.1"
+rmcp = "1.2.0"
 rmp-serde = "1.3.1"
 rolling-file = "0.2.0"
 rust-embed = "8.11.0"
@@ -241,7 +243,7 @@ secrecy = { version = "0.10", features = ["serde"] }
 send_wrapper = "0.6.0"
 serde = { version = "1.0.228", features = ["derive", "rc"] }
 serde_json = "1.0.149"
-serde_with = { version = "3.17.0", features = ["base64", "macros"] }
+serde_with = { version = "3.18.0", features = ["base64", "macros"] }
 serde_yaml_ng = "0.10.0"
 serial_test = "3.4.0"
 server = { path = "core/server" }
@@ -260,26 +262,26 @@ strum = { version = "0.28.0", features = ["derive"] }
 strum_macros = "0.28.0"
 syn = { version = "2", features = ["full", "extra-traits"] }
 sysinfo = "0.38.4"
-tempfile = "3.26.0"
+tempfile = "3.27.0"
 test-case = "3.3.1"
 testcontainers-modules = { version = "0.15.0", features = ["postgres"] }
 thiserror = "2.0.18"
 tokio = { version = "1.50.0", features = ["full"] }
 tokio-rustls = "0.26.4"
-tokio-tungstenite = { version = "0.28", features = ["rustls-tls-webpki-roots"] }
+tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] }
 tokio-util = { version = "0.7.18", features = ["compat"] }
-toml = "1.0.6"
+toml = "1.0.7"
 tower-http = { version = "0.6.8", features = ["add-extension", "cors", "trace"] }
 tracing = "0.1.44"
 tracing-appender = "0.2.4"
 tracing-opentelemetry = "0.32.1"
-tracing-subscriber = { version = "0.3.22", default-features = false, features = [
+tracing-subscriber = { version = "0.3.23", default-features = false, features = [
     "fmt",
     "env-filter",
     "ansi",
 ] }
 trait-variant = "0.1.2"
-tungstenite = "0.28.0"
+tungstenite = "0.29.0"
 twox-hash = { version = "2.1.2", features = ["xxhash32"] }
 ulid = "1.2.1"
 uuid = { version = "1.22.0", features = ["v4", "v7", "fast-rng", "serde", "zerocopy"] }
@@ -296,9 +298,9 @@ web-sys = { version = "0.3", features = [
     "ResizeObserverEntry",
 ] }
 webpki-roots = "1.0.6"
-yew = { version = "0.22", features = ["csr"] }
-yew-router = "0.19"
-zip = { version = "8.2.0", default-features = false, features = ["deflate"] }
+yew = { version = "0.23", features = ["csr"] }
+yew-router = "0.20"
+zip = { version = "8.3.0", default-features = false, features = ["deflate"] }
 
 [profile.release]
 lto = true

{apache_iggy-0.7.3.dev1 → apache_iggy-0.7.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-iggy
-Version: 0.7.3.dev1
+Version: 0.7.5.dev1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
@@ -46,7 +47,7 @@ License-File: LICENSE
 License-File: NOTICE
 Summary: Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second.
 Keywords: streaming,messaging,pubsub,iggy,rust,performance
-Author-email: Dario Lencina Talarico <darioalessandrolencina@gmail.com>, Albin Skott <albin@mattsson.io>
+Author-email: Iggy Committers <dev@iggy.apache.org>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Bug Tracker, https://github.com/apache/iggy/issues

{apache_iggy-0.7.3.dev1 → apache_iggy-0.7.5.dev1}/core/common/Cargo.toml

@@ -16,7 +16,7 @@
 # under the License.
 [package]
 name = "iggy_common"
-version = "0.9.3-edge.1"
+version = "0.9.5-edge.1"
 description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second."
 edition = "2024"
 license = "Apache-2.0"
@@ -30,13 +30,13 @@ readme = "README.md"
 [dependencies]
 aes-gcm = { workspace = true }
 ahash = { workspace = true }
+aligned-vec = { workspace = true }
 async-broadcast = { workspace = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
 blake3 = { workspace = true }
 bon = { workspace = true }
 byte-unit = { workspace = true }
-bytemuck = { workspace = true }
 bytes = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true }
@@ -44,7 +44,6 @@ comfy-table = { workspace = true }
 compio = { workspace = true }
 crossbeam = { workspace = true }
 derive_more = { workspace = true }
-enumset = { workspace = true }
 err_trail = { workspace = true }
 human-repr = { workspace = true }
 humantime = { workspace = true }

{apache_iggy-0.7.3.dev1/core/sdk → apache_iggy-0.7.5.dev1/core/common}/README.md

@@ -233,7 +233,7 @@ Run the tests:
 
 Set root user credentials (OPTIONAL):
 
-Iggy requires credentials to authenticate request to the server.
+Iggy requires credentials to authenticate requests to the server.
 You can set the root user **before** starting the server.
 
 (macOS/Linux)

{apache_iggy-0.7.3.dev1 → apache_iggy-0.7.5.dev1}/core/common/src/alloc/buffer.rs

@@ -16,11 +16,15 @@
  * under the License.
  */
 
-use super::memory_pool::{BytesMutExt, memory_pool};
-use bytes::{Buf, BufMut, Bytes, BytesMut};
+use crate::alloc::memory_pool::{ALIGNMENT, AlignedBuffer};
+
+use super::memory_pool::{AlignedBufferExt, memory_pool};
+use bytes::Bytes;
 use compio::buf::{IoBuf, IoBufMut, SetLen};
-use std::mem::MaybeUninit;
-use std::ops::{Deref, DerefMut};
+use std::{
+    mem::MaybeUninit,
+    ops::{Deref, DerefMut},
+};
 
 /// A buffer wrapper that participates in memory pooling.
 ///
@@ -32,7 +36,7 @@ pub struct PooledBuffer {
     from_pool: bool,
     original_capacity: usize,
     original_bucket_idx: Option<usize>,
-    inner: BytesMut,
+    inner: AlignedBuffer,
 }
 
 impl Default for PooledBuffer {
@@ -48,13 +52,21 @@ impl PooledBuffer {
     ///
     /// * `capacity` - The capacity of the buffer
     pub fn with_capacity(capacity: usize) -> Self {
-        let (buffer, was_pool_allocated) = memory_pool().acquire_buffer(capacity);
+        let (buffer, was_pool_allocated) = memory_pool().acquire_buffer(capacity.max(ALIGNMENT));
         let original_capacity = buffer.capacity();
         let original_bucket_idx = if was_pool_allocated {
             memory_pool().best_fit(original_capacity)
         } else {
             None
         };
+
+        debug_assert_eq!(
+            buffer.as_ptr() as usize % ALIGNMENT,
+            0,
+            "PooledBuffer not aligned to {} bytes",
+            ALIGNMENT
+        );
+
         Self {
             from_pool: was_pool_allocated,
             original_capacity,
@@ -63,12 +75,12 @@ impl PooledBuffer {
         }
     }
 
-    /// Creates a new pooled buffer from an existing `BytesMut`.
+    /// Creates a new pooled buffer from an existing `AlignedBuffer`.
     ///
     /// # Arguments
     ///
-    /// * `existing` - The existing `BytesMut` buffer
-    pub fn from_existing(existing: BytesMut) -> Self {
+    /// * `existing` - The existing `AlignedBuffer` buffer
+    pub fn from_existing(existing: AlignedBuffer) -> Self {
         Self {
             from_pool: false,
             original_capacity: existing.capacity(),
@@ -83,7 +95,7 @@ impl PooledBuffer {
             from_pool: false,
             original_capacity: 0,
             original_bucket_idx: None,
-            inner: BytesMut::new(),
+            inner: AlignedBuffer::new(ALIGNMENT),
         }
     }
 
@@ -127,6 +139,52 @@ impl PooledBuffer {
         }
     }
 
+    /// Split the buffer at given position, returning a new PooledBuffer
+    /// containing byte [0, at) and leaving [at, len)
+    ///
+    /// # Panic
+    /// Panics if at > len
+    pub fn split_to(&mut self, at: usize) -> PooledBuffer {
+        assert!(
+            at <= self.len(),
+            "split_to out of bounds: at={}, len={}",
+            at,
+            self.len()
+        );
+
+        let mut new_buff = PooledBuffer::with_capacity(at);
+        new_buff.inner.extend_from_slice(&self.inner[..at]);
+
+        // SAFETY:
+        // - `self.inner.as_ptr().add(at)` is valid for `new_len` because
+        // `at + new_len === old_len <= cap`. Similar with `self.inner.as_mut_ptr()`
+        //
+        // -  source range is `[at, at + new_len)` and the destination is
+        //   `[0, new_len)`.  These ranges do not overlap when `at > 0`.
+        // - when `at == 0`, the operation is noop
+        let new_len = self.len() - at;
+        if new_len > 0 {
+            unsafe {
+                // move [at..] to [0..]
+                std::ptr::copy(
+                    self.inner.as_ptr().add(at),
+                    self.inner.as_mut_ptr(),
+                    new_len,
+                );
+
+                self.inner.set_len(new_len);
+            }
+        } else {
+            self.inner.clear();
+        }
+
+        new_buff
+    }
+
+    pub fn put<T: AsRef<[u8]>>(&mut self, src: T) {
+        self.extend_from_slice(src.as_ref());
+    }
+
     /// Wrapper for extend_from_slice which might cause resize
     pub fn extend_from_slice(&mut self, extend_from: &[u8]) {
         let before_cap = self.inner.capacity();
@@ -140,7 +198,9 @@ impl PooledBuffer {
     /// Wrapper for put_bytes which might cause resize
     pub fn put_bytes(&mut self, byte: u8, len: usize) {
         let before_cap = self.inner.capacity();
-        self.inner.put_bytes(byte, len);
+
+        let start = self.inner.len();
+        self.inner.resize(start + len, byte);
 
         if self.inner.capacity() != before_cap {
             self.check_for_resize();
@@ -149,18 +209,18 @@ impl PooledBuffer {
 
     /// Wrapper for put_slice which might cause resize
     pub fn put_slice(&mut self, src: &[u8]) {
-        let before_cap = self.inner.capacity();
-        self.inner.put_slice(src);
-
-        if self.inner.capacity() != before_cap {
-            self.check_for_resize();
-        }
+        self.extend_from_slice(src);
+        // let before_cap = self.inner.capacity();
+        //
+        // if self.inner.capacity() != before_cap {
+        //     self.check_for_resize();
+        // }
     }
 
     /// Wrapper for put_u32_le which might cause resize
     pub fn put_u32_le(&mut self, value: u32) {
         let before_cap = self.inner.capacity();
-        self.inner.put_u32_le(value);
+        self.inner.extend_from_slice(&value.to_le_bytes());
 
         if self.inner.capacity() != before_cap {
             self.check_for_resize();
@@ -170,7 +230,7 @@ impl PooledBuffer {
     /// Wrapper for put_u64_le which might cause resize
     pub fn put_u64_le(&mut self, value: u64) {
         let before_cap = self.inner.capacity();
-        self.inner.put_u64_le(value);
+        self.inner.extend_from_slice(&value.to_le_bytes());
 
         if self.inner.capacity() != before_cap {
             self.check_for_resize();
@@ -192,11 +252,12 @@ impl PooledBuffer {
         self.inner.is_empty()
     }
 
-    /// Consumes the PooledBuffer and returns the inner BytesMut.
+    /// Consumes the PooledBuffer and returns the inner AlignedBuffer.
     /// Note: This bypasses pool return logic, use with caution.
-    pub fn into_inner(self) -> BytesMut {
+    pub fn into_inner(self) -> AlignedBuffer {
         let mut this = std::mem::ManuallyDrop::new(self);
-        std::mem::take(&mut this.inner)
+
+        std::mem::replace(&mut this.inner, AlignedBuffer::new(ALIGNMENT))
     }
 
     /// Freezes the buffer, converting it to an immutable `Bytes`.
@@ -205,24 +266,26 @@ impl PooledBuffer {
     /// return memory to the pool on drop (the frozen Bytes owns the allocation).
     /// The returned `Bytes` is Arc-backed, allowing cheap clones.
     pub fn freeze(&mut self) -> Bytes {
-        // Decrement pool counter since memory is transferred to Bytes
-        // and won't be returned to the pool.
+        let buf = std::mem::replace(&mut self.inner, AlignedBuffer::new(ALIGNMENT));
+
+        // Update pool accounting
         if self.from_pool
             && let Some(bucket_idx) = self.original_bucket_idx
         {
             memory_pool().dec_bucket_in_use(bucket_idx);
         }
-
-        let inner = std::mem::take(&mut self.inner);
         self.from_pool = false;
         self.original_capacity = 0;
         self.original_bucket_idx = None;
-        inner.freeze()
+
+        // Zero copy: Bytes takes ownership of the AlignedBuffer
+        // and will drop it when refcount reaches zero
+        Bytes::from_owner(buf)
     }
 }
 
 impl Deref for PooledBuffer {
-    type Target = BytesMut;
+    type Target = AlignedBuffer;
 
     fn deref(&self) -> &Self::Target {
         &self.inner
@@ -238,7 +301,7 @@ impl DerefMut for PooledBuffer {
 impl Drop for PooledBuffer {
     fn drop(&mut self) {
         if self.from_pool {
-            let buf = std::mem::take(&mut self.inner);
+            let buf = std::mem::replace(&mut self.inner, AlignedBuffer::new(ALIGNMENT));
             buf.return_to_pool(self.original_capacity, true);
         }
     }
@@ -252,27 +315,15 @@ impl From<&[u8]> for PooledBuffer {
     }
 }
 
-impl From<BytesMut> for PooledBuffer {
-    fn from(bytes: BytesMut) -> Self {
-        Self::from_existing(bytes)
+impl AsRef<[u8]> for PooledBuffer {
+    fn as_ref(&self) -> &[u8] {
+        &self.inner
     }
 }
 
-impl Buf for PooledBuffer {
-    fn remaining(&self) -> usize {
-        self.inner.remaining()
-    }
-
-    fn chunk(&self) -> &[u8] {
-        self.inner.chunk()
-    }
-
-    fn advance(&mut self, cnt: usize) {
-        Buf::advance(&mut self.inner, cnt)
-    }
-
-    fn chunks_vectored<'t>(&'t self, dst: &mut [std::io::IoSlice<'t>]) -> usize {
-        self.inner.chunks_vectored(dst)
+impl From<AlignedBuffer> for PooledBuffer {
+    fn from(buffer: AlignedBuffer) -> Self {
+        Self::from_existing(buffer)
     }
 }