icechunk 1.1.16__tar.gz → 1.1.18__tar.gz

{icechunk-1.1.16 → icechunk-1.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: icechunk
-Version: 1.1.16
+Version: 1.1.18
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk"
-version = "0.3.19"
+version = "0.3.21"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "README.md"
 repository = "https://github.com/earth-mover/icechunk"
@@ -40,10 +40,11 @@ rmp-serde = "1.3.1"
 url = "2.5.7"
 async-stream = "0.3.6"
 rmpv = { version = "1.3.1", features = ["serde", "with-serde"] }
-aws-sdk-s3 = "1.119.0"
+aws-sdk-s3 = "1.120.0"
 aws-config = "1.8.12"
 aws-credential-types = "1.2.11"
 typed-path = "0.12.0"
+aws-smithy-runtime = "1.8.5"
 aws-smithy-types-convert = { version = "0.60.11", features = [
   "convert-chrono",
   "convert-streams",

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk/src/change_set.rs

@@ -6,6 +6,7 @@ use std::{
 use bytes::Bytes;
 use itertools::{Either, Itertools as _};
 use serde::{Deserialize, Serialize};
+use tracing::warn;
 
 use crate::{
     format::{
@@ -16,6 +17,13 @@ use crate::{
     session::{SessionResult, find_coord},
 };
 
+// We have limitations on how many chunks we can save on a single commit.
+// Mostly due to flatbuffers not supporting 64-bit offsets,
+// but also because we can't do transaction log splitting (like we can with manifests).
+// For now we suggest smaller commits as a solution.
+// See discussion in https://github.com/earth-mover/icechunk/issues/1558 for more details
+const NUM_CHUNKS_LIMIT: u64 = 50_000_000;
+
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ArrayData {
     pub shape: ArrayShape,
@@ -31,6 +39,13 @@ pub struct ChangeSet {
     updated_arrays: HashMap<NodeId, ArrayData>,
     updated_groups: HashMap<NodeId, Bytes>,
     set_chunks: BTreeMap<NodeId, HashMap<ManifestExtents, SplitManifest>>,
+
+    // Number of chunks added to this change set
+    num_chunks: u64,
+
+    // Did we already print a warning about too many chunks in this change set?
+    excessive_num_chunks_warned: bool,
+
     // This map keeps track of any chunk deletes that are
     // outside the domain of the current array shape. This is needed to handle
     // the very unlikely case of multiple resizes in the same session.
@@ -244,7 +259,8 @@ impl ChangeSet {
         let extent = splits.find(&coord).expect("logic bug. Trying to set chunk ref but can't find the appropriate split manifest.");
         // this implementation makes delete idempotent
         // it allows deleting a deleted chunk by repeatedly setting None.
-        self.set_chunks
+        let old = self
+            .set_chunks
             .entry(node_id)
             .or_insert_with(|| {
                 HashMap::<
@@ -255,6 +271,18 @@ impl ChangeSet {
             .entry(extent.clone())
             .or_default()
             .insert(coord, data);
+
+        if old.is_none() {
+            self.num_chunks += 1;
+        }
+
+        if self.num_chunks > NUM_CHUNKS_LIMIT && !self.excessive_num_chunks_warned {
+            warn!(
+                "There are more than {NUM_CHUNKS_LIMIT} chunk references being loaded into this commit. This is close to the maximum number of chunk modifications Icechunk supports in a single commit, we recommend to split into smaller commits."
+            );
+
+            self.excessive_num_chunks_warned = true;
+        }
     }
 
     pub fn get_chunk_ref(

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk/src/config.rs

@@ -450,6 +450,8 @@ impl RepositoryConfig {
                 (Some(s), None) => Some(s.clone()),
                 (Some(mine), Some(theirs)) => Some(mine.merge(theirs)),
             },
+            // For virtual_chunk_containers, replace rather than extend.
+            // This is consistent with other fields: if specified, it replaces.
             virtual_chunk_containers: match (
                 &self.virtual_chunk_containers,
                 other.virtual_chunk_containers,
@@ -457,11 +459,7 @@ impl RepositoryConfig {
                 (None, None) => None,
                 (None, Some(c)) => Some(c),
                 (Some(c), None) => Some(c.clone()),
-                (Some(mine), Some(theirs)) => {
-                    let mut merged = mine.clone();
-                    merged.extend(theirs);
-                    Some(merged)
-                }
+                (Some(_), Some(theirs)) => Some(theirs),
             },
             manifest: match (&self.manifest, other.manifest) {
                 (None, None) => None,
@@ -678,4 +676,89 @@ mod tests {
             assert_eq!(config, roundtrip);
         }
     }
+
+    #[icechunk_macros::test]
+    fn test_merge_replaces_virtual_chunk_containers() {
+        // Create a config with a VCC
+        let mut config1 = RepositoryConfig::default();
+        config1.set_virtual_chunk_container(
+            VirtualChunkContainer::new(
+                "s3://bucket1/".to_string(),
+                ObjectStoreConfig::S3(S3Options {
+                    region: Some("us-east-1".to_string()),
+                    endpoint_url: None,
+                    anonymous: false,
+                    allow_http: false,
+                    force_path_style: false,
+                    network_stream_timeout_seconds: None,
+                    requester_pays: false,
+                }),
+            )
+            .unwrap(),
+        );
+
+        // Create a config with no VCCs (cleared)
+        let mut config2 = RepositoryConfig::default();
+        config2.clear_virtual_chunk_containers();
+
+        // Merge: config2 (empty VCCs) should replace config1's VCCs
+        let merged = config1.merge(config2);
+
+        // VCCs should be empty after merge
+        assert_eq!(
+            merged.virtual_chunk_containers,
+            Some(std::collections::HashMap::new()),
+            "Merging with cleared VCCs should result in empty VCCs"
+        );
+    }
+
+    #[icechunk_macros::test]
+    fn test_merge_replaces_virtual_chunk_containers_with_new_ones() {
+        // Create a config with VCC1
+        let mut config1 = RepositoryConfig::default();
+        config1.set_virtual_chunk_container(
+            VirtualChunkContainer::new(
+                "s3://bucket1/".to_string(),
+                ObjectStoreConfig::S3(S3Options {
+                    region: Some("us-east-1".to_string()),
+                    endpoint_url: None,
+                    anonymous: false,
+                    allow_http: false,
+                    force_path_style: false,
+                    network_stream_timeout_seconds: None,
+                    requester_pays: false,
+                }),
+            )
+            .unwrap(),
+        );
+
+        // Create a config with VCC2
+        let mut config2 = RepositoryConfig::default();
+        config2.set_virtual_chunk_container(
+            VirtualChunkContainer::new(
+                "s3://bucket2/".to_string(),
+                ObjectStoreConfig::S3(S3Options {
+                    region: Some("us-west-2".to_string()),
+                    endpoint_url: None,
+                    anonymous: false,
+                    allow_http: false,
+                    force_path_style: false,
+                    network_stream_timeout_seconds: None,
+                    requester_pays: false,
+                }),
+            )
+            .unwrap(),
+        );
+
+        // Merge: config2's VCCs should replace config1's VCCs
+        let merged = config1.merge(config2);
+
+        let vccs = merged.virtual_chunk_containers.unwrap();
+        assert_eq!(vccs.len(), 1, "Should have exactly one VCC after merge");
+        assert!(
+            vccs.contains_key("s3://bucket2/"),
+            "Should contain bucket2, not bucket1"
+        );
+        assert!(!vccs.contains_key("s3://bucket1/"), "Should not contain bucket1");
+    }
 }

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk/src/session.rs

@@ -945,7 +945,8 @@ impl Session {
         if self.read_only() {
             return Err(SessionErrorKind::ReadOnlySession.into());
         }
-        let Session { splits: other_splits, change_set, .. } = other;
+        let Session { splits: other_splits, change_set, config: other_config, .. } =
+            other;
 
         if self.splits.iter().any(|(node, our_splits)| {
             other_splits
@@ -953,7 +954,7 @@ impl Session {
                 .is_some_and(|their_splits| !our_splits.compatible_with(their_splits))
         }) {
             let ours = self.config().manifest().splitting().clone();
-            let theirs = self.config().manifest().splitting().clone();
+            let theirs = other_config.manifest().splitting().clone();
             return Err(
                 SessionErrorKind::IncompatibleSplittingConfig { ours, theirs }.into()
             );

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk/src/storage/s3.rs

@@ -32,6 +32,7 @@ use aws_sdk_s3::{
     primitives::ByteStream,
     types::{CompletedMultipartUpload, CompletedPart, Delete, Object, ObjectIdentifier},
 };
+use aws_smithy_runtime::client::retries::classifiers::HttpStatusCodeClassifier;
 use aws_smithy_types_convert::{date_time::DateTimeExt, stream::PaginationStreamExt};
 use bytes::{Buf, Bytes};
 use chrono::{DateTime, Utc};
@@ -137,9 +138,8 @@ pub async fn mk_client(
 
     #[allow(clippy::unwrap_used)]
     let app_name = AppName::new("icechunk").unwrap();
-    let mut aws_config = aws_config::defaults(BehaviorVersion::v2025_08_07())
-        .region(region)
-        .app_name(app_name);
+    let mut aws_config =
+        aws_config::defaults(BehaviorVersion::latest()).region(region).app_name(app_name);
 
     if let Some(endpoint) = endpoint {
         aws_config = aws_config.endpoint_url(endpoint)
@@ -196,6 +196,18 @@ pub async fn mk_client(
 
     s3_builder = s3_builder.identity_cache(id_cache);
 
+    // Add retry classifier for HTTP 408 (Request Timeout)
+    // The default HttpStatusCodeClassifier only retries on 500, 502, 503, 504
+    static RETRY_CODES: &[u16] = &[408];
+    // This confusingly named `retry_classifier` method ends up calling
+    // `push_retry_classifier` after wrapping our custom classifier in `SharedRetryClassifier`.
+    // Ultimately, this is a push on to a `Vec<SharedRetryClassifier>`, and is thus additive
+    // to the existing default retry configuration.
+    // https://github.com/smithy-lang/smithy-rs/blob/cfcc39cf4b5bea665bba684b64bfca2b89e4bc73/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs#L755
+    // https://github.com/smithy-lang/smithy-rs/blob/cfcc39cf4b5bea665bba684b64bfca2b89e4bc73/rust-runtime/aws-smithy-runtime-api/src/client/runtime_components.rs#L370
+    s3_builder = s3_builder
+        .retry_classifier(HttpStatusCodeClassifier::new_from_codes(RETRY_CODES));
+
     if !extra_read_headers.is_empty() || !extra_write_headers.is_empty() {
         s3_builder = s3_builder.interceptor(ExtraHeadersInterceptor {
             extra_read_headers,

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk-python"
-version = "1.1.16"
+version = "1.1.18"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "README.md"
 repository = "https://github.com/earth-mover/icechunk"
@@ -21,7 +21,7 @@ crate-type = ["cdylib"]
 bytes = "1.11.0"
 chrono = { version = "0.4.42" }
 futures = "0.3.31"
-icechunk = { path = "../icechunk", version = "0.3.19", features = ["logs"] }
+icechunk = { path = "../icechunk", version = "0.3.21", features = ["logs"] }
 itertools = "0.14.0"
 pyo3 = { version = "0.27.2", features = [
   "chrono",

{icechunk-1.1.16 → icechunk-1.1.18}/icechunk-python/tests/test_config.py

@@ -142,6 +142,39 @@ def test_virtual_chunk_containers() -> None:
     assert config.virtual_chunk_containers["s3://testbucket/"] == container
 
 
+def test_clear_virtual_chunk_containers_persists_through_reopen() -> None:
+    """Test that clearing VCCs is respected by reopen().
+
+    Regression test for: https://github.com/earth-mover/icechunk/issues/XXX
+    """
+    storage = icechunk.in_memory_storage()
+
+    # Create repo and add a VCC
+    repo = icechunk.Repository.create(storage=storage)
+    config = repo.config
+    store_config = icechunk.s3_store(region="us-east-1")
+    container = icechunk.VirtualChunkContainer("s3://testbucket/", store_config)
+    config.set_virtual_chunk_container(container)
+    repo = repo.reopen(config=config)
+    repo.save_config()
+
+    vccs = repo.config.virtual_chunk_containers or {}
+    assert "s3://testbucket/" in vccs
+
+    # Clear VCCs and reopen
+    config = repo.config
+    config.clear_virtual_chunk_containers()
+    assert config.virtual_chunk_containers == {}
+
+    repo = repo.reopen(config=config)
+
+    # VCCs should be cleared after reopen
+    reopened_vccs = repo.config.virtual_chunk_containers or {}
+    assert (
+        reopened_vccs == {}
+    ), f"Expected no VCCs after reopen, got: {list(reopened_vccs.keys())}"
+
+
 def test_can_change_deep_config_values() -> None:
     storage = icechunk.in_memory_storage()
     repo = icechunk.Repository.create(