icechunk 0.2.3__tar.gz → 0.2.4__tar.gz

{icechunk-0.2.3 → icechunk-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: icechunk
-Version: 0.2.3
+Version: 0.2.4
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{icechunk-0.2.3 → icechunk-0.2.4}/icechunk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk"
-version = "0.2.3"
+version = "0.2.4"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "README.md"
 repository = "https://github.com/earth-mover/icechunk"

{icechunk-0.2.3 → icechunk-0.2.4}/icechunk/src/storage/mod.rs

@@ -712,10 +712,27 @@ pub async fn new_gcs_storage(
 #[allow(clippy::unwrap_used, clippy::panic)]
 mod tests {
 
-    use std::collections::HashSet;
+    use std::{collections::HashSet, fs::File, io::Write, path::PathBuf};
 
     use super::*;
     use proptest::prelude::*;
+    use tempfile::TempDir;
+
+    #[tokio::test]
+    async fn test_is_clean() {
+        let repo_dir = TempDir::new().unwrap();
+        let s = new_local_filesystem_storage(repo_dir.path()).await.unwrap();
+        assert!(s.root_is_clean().await.unwrap());
+
+        let mut file = File::create(repo_dir.path().join("foo.txt")).unwrap();
+        write!(file, "hello").unwrap();
+        assert!(!s.root_is_clean().await.unwrap());
+
+        let inside_existing =
+            PathBuf::from_iter([repo_dir.path().as_os_str().to_str().unwrap(), "foo"]);
+        let s = new_local_filesystem_storage(&inside_existing).await.unwrap();
+        assert!(s.root_is_clean().await.unwrap());
+    }
 
     proptest! {
         #![proptest_config(ProptestConfig {

{icechunk-0.2.3 → icechunk-0.2.4}/icechunk/src/storage/object_store.rs

@@ -83,7 +83,6 @@ impl ObjectStorage {
             Arc::new(LocalFileSystemObjectStoreBackend { path: prefix.to_path_buf() });
         let client = backend.mk_object_store().await?;
         let storage = ObjectStorage { backend, client: OnceCell::new_with(Some(client)) };
-
         Ok(storage)
     }
 
@@ -1022,6 +1021,8 @@ mod tests {
 
     use tempfile::TempDir;
 
+    use crate::format::{ChunkId, ManifestId, SnapshotId};
+
     use super::ObjectStorage;
 
     #[tokio::test]
@@ -1071,4 +1072,34 @@ mod tests {
             ObjectStorage::new_local_filesystem(PathBuf::from(&rel_path).as_path()).await;
         assert!(store.is_ok());
     }
+
+    #[tokio::test]
+    async fn test_object_store_paths() {
+        let store = ObjectStorage::new_local_filesystem(PathBuf::from(".").as_path())
+            .await
+            .unwrap();
+
+        let ref_key = "ref_key";
+        let ref_path = store.ref_key(ref_key);
+        assert_eq!(ref_path.to_string(), format!("refs/{ref_key}"));
+
+        let snapshot_id = SnapshotId::random();
+        let snapshot_path = store.get_snapshot_path(&snapshot_id);
+        assert_eq!(snapshot_path.to_string(), format!("snapshots/{snapshot_id}"));
+
+        let manifest_id = ManifestId::random();
+        let manifest_path = store.get_manifest_path(&manifest_id);
+        assert_eq!(manifest_path.to_string(), format!("manifests/{manifest_id}"));
+
+        let chunk_id = ChunkId::random();
+        let chunk_path = store.get_chunk_path(&chunk_id);
+        assert_eq!(chunk_path.to_string(), format!("chunks/{chunk_id}"));
+
+        let transaction_id = SnapshotId::random();
+        let transaction_path = store.get_transaction_path(&transaction_id);
+        assert_eq!(
+            transaction_path.to_string(),
+            format!("transactions/{transaction_id}")
+        );
+    }
 }

{icechunk-0.2.3 → icechunk-0.2.4}/icechunk/src/storage/s3.rs

@@ -30,7 +30,6 @@ use aws_sdk_s3::{
 use aws_smithy_types_convert::{date_time::DateTimeExt, stream::PaginationStreamExt};
 use bytes::{Buf, Bytes};
 use chrono::{DateTime, Utc};
-use err_into::ErrorInto as _;
 use futures::{
     stream::{self, BoxStream},
     StreamExt, TryStreamExt,
@@ -150,10 +149,10 @@ impl S3Storage {
 
     fn get_path_str(&self, file_prefix: &str, id: &str) -> StorageResult<String> {
         let path = PathBuf::from_iter([self.prefix.as_str(), file_prefix, id]);
-        path.into_os_string()
-            .into_string()
-            .map_err(StorageErrorKind::BadPrefix)
-            .err_into()
+        let path_str =
+            path.into_os_string().into_string().map_err(StorageErrorKind::BadPrefix)?;
+
+        Ok(path_str.replace("\\", "/"))
     }
 
     fn get_path<const SIZE: usize, T: FileTypeTag>(
@@ -187,10 +186,10 @@ impl S3Storage {
 
     fn ref_key(&self, ref_key: &str) -> StorageResult<String> {
         let path = PathBuf::from_iter([self.prefix.as_str(), REF_PREFIX, ref_key]);
-        path.into_os_string()
-            .into_string()
-            .map_err(StorageErrorKind::BadPrefix)
-            .err_into()
+        let path_str =
+            path.into_os_string().into_string().map_err(StorageErrorKind::BadPrefix)?;
+
+        Ok(path_str.replace("\\", "/"))
     }
 
     async fn get_object_reader(
@@ -612,10 +611,7 @@ impl Storage for S3Storage {
         _settings: &Settings,
         prefix: &str,
     ) -> StorageResult<BoxStream<'a, StorageResult<ListInfo<String>>>> {
-        let prefix = PathBuf::from_iter([self.prefix.as_str(), prefix])
-            .into_os_string()
-            .into_string()
-            .map_err(StorageErrorKind::BadPrefix)?;
+        let prefix = format!("{}/{}", self.prefix, prefix).replace("//", "/");
         let stream = self
             .get_client()
             .await
@@ -798,4 +794,39 @@ mod tests {
         let deserialized: S3Storage = serde_json::from_str(&serialized).unwrap();
         assert_eq!(storage.config, deserialized.config);
     }
+
+    #[tokio::test]
+    async fn test_s3_paths() {
+        let storage = S3Storage::new(
+            S3Options {
+                region: Some("us-west-2".to_string()),
+                endpoint_url: None,
+                allow_http: true,
+                anonymous: false,
+            },
+            "bucket".to_string(),
+            Some("prefix".to_string()),
+            S3Credentials::FromEnv,
+        )
+        .unwrap();
+
+        let ref_path = storage.ref_key("ref_key").unwrap();
+        assert_eq!(ref_path, "prefix/refs/ref_key");
+
+        let snapshot_id = SnapshotId::random();
+        let snapshot_path = storage.get_snapshot_path(&snapshot_id).unwrap();
+        assert_eq!(snapshot_path, format!("prefix/snapshots/{snapshot_id}"));
+
+        let manifest_id = ManifestId::random();
+        let manifest_path = storage.get_manifest_path(&manifest_id).unwrap();
+        assert_eq!(manifest_path, format!("prefix/manifests/{manifest_id}"));
+
+        let chunk_id = ChunkId::random();
+        let chunk_path = storage.get_chunk_path(&chunk_id).unwrap();
+        assert_eq!(chunk_path, format!("prefix/chunks/{chunk_id}"));
+
+        let transaction_id = SnapshotId::random();
+        let transaction_path = storage.get_transaction_path(&transaction_id).unwrap();
+        assert_eq!(transaction_path, format!("prefix/transactions/{transaction_id}"));
+    }
 }

{icechunk-0.2.3 → icechunk-0.2.4}/icechunk-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk-python"
-version = "0.2.3"
+version = "0.2.4"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "../README.md"
 repository = "https://github.com/earth-mover/icechunk"
@@ -21,7 +21,7 @@ crate-type = ["cdylib"]
 bytes = "1.9.0"
 chrono = { version = "0.4.39" }
 futures = "0.3.31"
-icechunk = { path = "../icechunk", version = "0.2.3", features = ["logs"] }
+icechunk = { path = "../icechunk", version = "0.2.4", features = ["logs"] }
 itertools = "0.14.0"
 pyo3 = { version = "0.23", features = [
   "chrono",

icechunk-0.2.4/icechunk-python/examples/mpwrite.py

@@ -0,0 +1,48 @@
+# An example of using multiprocessing to write to an Icechunk dataset
+
+import tempfile
+from concurrent.futures import ProcessPoolExecutor
+
+import xarray as xr
+from icechunk import Repository, Session, local_filesystem_storage
+from icechunk.distributed import merge_sessions
+
+
+def write_timestamp(*, itime: int, session: Session) -> Session:
+    # pass a list to isel to preserve the time dimension
+    ds = xr.tutorial.open_dataset("rasm").isel(time=[itime])
+    # region="auto" tells Xarray to infer which "region" of the output arrays to write to.
+    ds.to_zarr(session.store, region="auto", consolidated=False)
+    return session
+
+
+if __name__ == "__main__":
+    ds = xr.tutorial.open_dataset("rasm").isel(time=slice(24))
+    repo = Repository.create(local_filesystem_storage(tempfile.mkdtemp()))
+    session = repo.writable_session("main")
+
+    chunks = {1 if dim == "time" else ds.sizes[dim] for dim in ds.Tair.dims}
+    ds.to_zarr(
+        session.store, compute=False, encoding={"Tair": {"chunks": chunks}}, mode="w"
+    )
+    # this commit is optional, but may be useful in your workflow
+    session.commit("initialize store")
+
+    session = repo.writable_session("main")
+    with ProcessPoolExecutor() as executor:
+        # opt-in to successful pickling of a writable session
+        with session.allow_pickling():
+            # submit the writes
+            futures = [
+                executor.submit(write_timestamp, itime=i, session=session)
+                for i in range(ds.sizes["time"])
+            ]
+            # grab the Session objects from each individual write task
+            sessions = [f.result() for f in futures]
+
+    # manually merge the remote sessions in to the local session
+    session = merge_sessions(session, *sessions)
+    session.commit("finished writes")
+
+    ondisk = xr.open_zarr(repo.readonly_session("main").store, consolidated=False)
+    xr.testing.assert_identical(ds, ondisk)