icechunk 0.2.2__tar.gz → 0.2.3__tar.gz

{icechunk-0.2.2 → icechunk-0.2.3}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: icechunk
-Version: 0.2.2
+Version: 0.2.3
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: zarr>=3
+Requires-Dist: zarr>=3,!=3.0.3
 Requires-Dist: boto3 ; extra == 'test'
 Requires-Dist: coverage ; extra == 'test'
 Requires-Dist: mypy ; extra == 'test'

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk"
-version = "0.2.2"
+version = "0.2.3"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "README.md"
 repository = "https://github.com/earth-mover/icechunk"

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/config.rs

@@ -24,6 +24,19 @@ pub struct S3Options {
     pub allow_http: bool,
 }
 
+impl fmt::Display for S3Options {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "S3Options(region={}, endpoint_url={}, anonymous={}, allow_http={})",
+            self.region.as_deref().unwrap_or("None"),
+            self.endpoint_url.as_deref().unwrap_or("None"),
+            self.anonymous,
+            self.allow_http
+        )
+    }
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum ObjectStoreConfig {

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/repository.rs

@@ -331,6 +331,16 @@ impl Repository {
         )
     }
 
+    #[instrument(skip(bytes))]
+    pub fn from_bytes(bytes: Vec<u8>) -> RepositoryResult<Self> {
+        rmp_serde::from_slice(&bytes).err_into()
+    }
+
+    #[instrument(skip(self))]
+    pub fn as_bytes(&self) -> RepositoryResult<Vec<u8>> {
+        rmp_serde::to_vec(self).err_into()
+    }
+
     #[instrument(skip_all)]
     pub async fn fetch_config(
         storage: &(dyn Storage + Send + Sync),

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/storage/logging.rs

@@ -1,4 +1,5 @@
 use std::{
+    fmt,
     ops::Range,
     sync::{Arc, Mutex},
 };
@@ -37,6 +38,12 @@ impl LoggingStorage {
     }
 }
 
+impl fmt::Display for LoggingStorage {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "LoggingStorage(backend={})", self.backend)
+    }
+}
+
 impl private::Sealed for LoggingStorage {}
 
 #[async_trait]

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/storage/mod.rs

@@ -298,7 +298,7 @@ pub enum WriteRefResult {
 /// Implementations are free to assume files are never overwritten.
 #[async_trait]
 #[typetag::serde(tag = "type")]
-pub trait Storage: fmt::Debug + private::Sealed + Sync + Send {
+pub trait Storage: fmt::Debug + fmt::Display + private::Sealed + Sync + Send {
     fn default_settings(&self) -> Settings {
         Default::default()
     }

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/storage/object_store.rs

@@ -27,7 +27,7 @@ use object_store::{
 use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
-    fmt::Debug,
+    fmt::{self, Debug, Display},
     fs::create_dir_all,
     future::ready,
     num::{NonZeroU16, NonZeroU64},
@@ -286,6 +286,12 @@ impl ObjectStorage {
     }
 }
 
+impl fmt::Display for ObjectStorage {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "ObjectStorage(backend={})", self.backend)
+    }
+}
+
 impl private::Sealed for ObjectStorage {}
 
 #[async_trait]
@@ -622,7 +628,7 @@ impl Storage for ObjectStorage {
 
 #[async_trait]
 #[typetag::serde(tag = "object_store_provider_type")]
-pub trait ObjectStoreBackend: Debug + Sync + Send {
+pub trait ObjectStoreBackend: Debug + Display + Sync + Send {
     async fn mk_object_store(&self) -> Result<Arc<dyn ObjectStore>, StorageError>;
 
     /// The prefix for the object store.
@@ -640,6 +646,12 @@ pub trait ObjectStoreBackend: Debug + Sync + Send {
 #[derive(Debug, Serialize, Deserialize)]
 pub struct InMemoryObjectStoreBackend;
 
+impl fmt::Display for InMemoryObjectStoreBackend {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "InMemoryObjectStoreBackend")
+    }
+}
+
 #[async_trait]
 #[typetag::serde(name = "in_memory_object_store_provider")]
 impl ObjectStoreBackend for InMemoryObjectStoreBackend {
@@ -672,6 +684,12 @@ pub struct LocalFileSystemObjectStoreBackend {
     path: PathBuf,
 }
 
+impl fmt::Display for LocalFileSystemObjectStoreBackend {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "LocalFileSystemObjectStoreBackend(path={})", self.path.display())
+    }
+}
+
 #[async_trait]
 #[typetag::serde(name = "local_file_system_object_store_provider")]
 impl ObjectStoreBackend for LocalFileSystemObjectStoreBackend {
@@ -720,6 +738,18 @@ pub struct S3ObjectStoreBackend {
     config: Option<S3Options>,
 }
 
+impl fmt::Display for S3ObjectStoreBackend {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "S3ObjectStoreBackend(bucket={}, prefix={}, config={})",
+            self.bucket,
+            self.prefix.as_deref().unwrap_or(""),
+            self.config.as_ref().map(|c| c.to_string()).unwrap_or("None".to_string())
+        )
+    }
+}
+
 #[async_trait]
 #[typetag::serde(name = "s3_object_store_provider")]
 impl ObjectStoreBackend for S3ObjectStoreBackend {
@@ -794,6 +824,18 @@ pub struct AzureObjectStoreBackend {
     config: Option<HashMap<AzureConfigKey, String>>,
 }
 
+impl fmt::Display for AzureObjectStoreBackend {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "AzureObjectStoreBackend(account={}, container={}, prefix={})",
+            self.account,
+            self.container,
+            self.prefix.as_deref().unwrap_or("")
+        )
+    }
+}
+
 #[async_trait]
 #[typetag::serde(name = "azure_object_store_provider")]
 impl ObjectStoreBackend for AzureObjectStoreBackend {
@@ -846,6 +888,17 @@ pub struct GcsObjectStoreBackend {
     config: Option<HashMap<GoogleConfigKey, String>>,
 }
 
+impl fmt::Display for GcsObjectStoreBackend {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "GcsObjectStoreBackend(bucket={}, prefix={})",
+            self.bucket,
+            self.prefix.as_deref().unwrap_or("")
+        )
+    }
+}
+
 #[async_trait]
 #[typetag::serde(name = "gcs_object_store_provider")]
 impl ObjectStoreBackend for GcsObjectStoreBackend {

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk/src/storage/s3.rs

@@ -1,4 +1,5 @@
 use std::{
+    fmt,
     future::ready,
     ops::Range,
     path::{Path, PathBuf},
@@ -59,6 +60,16 @@ pub struct S3Storage {
     client: OnceCell<Arc<Client>>,
 }
 
+impl fmt::Display for S3Storage {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "S3Storage(bucket={}, prefix={}, config={})",
+            self.bucket, self.prefix, self.config,
+        )
+    }
+}
+
 #[instrument(skip(credentials))]
 pub async fn mk_client(config: &S3Options, credentials: S3Credentials) -> Client {
     let region = config

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk-python"
-version = "0.2.2"
+version = "0.2.3"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "../README.md"
 repository = "https://github.com/earth-mover/icechunk"
@@ -21,7 +21,7 @@ crate-type = ["cdylib"]
 bytes = "1.9.0"
 chrono = { version = "0.4.39" }
 futures = "0.3.31"
-icechunk = { path = "../icechunk", version = "0.2.2", features = ["logs"] }
+icechunk = { path = "../icechunk", version = "0.2.3", features = ["logs"] }
 itertools = "0.14.0"
 pyo3 = { version = "0.23", features = [
   "chrono",

{icechunk-0.2.2 → icechunk-0.2.3/icechunk-python}/python/icechunk/__init__.py

@@ -52,6 +52,7 @@ from icechunk.credentials import (
     containers_credentials,
     gcs_credentials,
     gcs_from_env_credentials,
+    gcs_refreshable_credentials,
     gcs_static_credentials,
     s3_anonymous_credentials,
     s3_credentials,
@@ -128,11 +129,13 @@ __all__ = [
     "containers_credentials",
     "gcs_credentials",
     "gcs_from_env_credentials",
+    "gcs_refreshable_credentials",
     "gcs_static_credentials",
     "gcs_storage",
     "in_memory_storage",
     "initialize_logs",
     "local_filesystem_storage",
+    "print_debug_info",
     "s3_anonymous_credentials",
     "s3_credentials",
     "s3_from_env_credentials",
@@ -144,4 +147,19 @@ __all__ = [
     "tigris_storage",
 ]
 
+
+def print_debug_info() -> None:
+    import platform
+    from importlib import import_module
+
+    print(f"platform:  {platform.platform()}")
+    print(f"python:  {platform.python_version()}")
+    print(f"icechunk:  {__version__}")
+    for package in ["zarr", "numcodecs", "xarray", "virtualizarr"]:
+        try:
+            print(f"{package}:  {import_module(package).__version__}")
+        except ModuleNotFoundError:
+            continue
+
+
 initialize_logs()

{icechunk-0.2.2 → icechunk-0.2.3/icechunk-python}/python/icechunk/_icechunk_python.pyi

@@ -947,6 +947,9 @@ class PyRepository:
     ) -> PyRepository: ...
     @staticmethod
     def exists(storage: Storage) -> bool: ...
+    @classmethod
+    def from_bytes(cls, data: bytes) -> PyRepository: ...
+    def as_bytes(self) -> bytes: ...
     @staticmethod
     def fetch_config(storage: Storage) -> RepositoryConfig | None: ...
     def save_config(self) -> None: ...
@@ -1418,6 +1421,7 @@ class Storage:
         *,
         config: dict[str, str] | None = None,
     ) -> Storage: ...
+    def __repr__(self) -> str: ...
     def default_settings(self) -> StorageSettings: ...
 
 class VersionSelection(Enum):

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk-python/python/icechunk/repository.py

@@ -140,6 +140,16 @@ class Repository:
         """
         return PyRepository.exists(storage)
 
+    def __getstate__(self) -> object:
+        return {
+            "_repository": self._repository.as_bytes(),
+        }
+
+    def __setstate__(self, state: object) -> None:
+        if not isinstance(state, dict):
+            raise ValueError("Invalid repository state")
+        self._repository = PyRepository.from_bytes(state["_repository"])
+
     @staticmethod
     def fetch_config(storage: Storage) -> RepositoryConfig | None:
         """

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk-python/src/config.rs

@@ -1312,6 +1312,10 @@ impl PyStorage {
         })
     }
 
+    pub fn __repr__(&self) -> String {
+        format!("{}", self.0)
+    }
+
     pub fn default_settings(&self) -> PyStorageSettings {
         self.0.default_settings().into()
     }

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk-python/src/repository.rs

@@ -1,4 +1,5 @@
 use std::{
+    borrow::Cow,
     collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     sync::Arc,
 };
@@ -463,6 +464,29 @@ impl PyRepository {
         })
     }
 
+    #[classmethod]
+    fn from_bytes(
+        _cls: Bound<'_, PyType>,
+        py: Python<'_>,
+        bytes: Vec<u8>,
+    ) -> PyResult<Self> {
+        // This is a compute intensive task, we need to release the Gil
+        py.allow_threads(move || {
+            let repository = Repository::from_bytes(bytes)
+                .map_err(PyIcechunkStoreError::RepositoryError)?;
+            Ok(Self(Arc::new(repository)))
+        })
+    }
+
+    fn as_bytes(&self, py: Python<'_>) -> PyResult<Cow<[u8]>> {
+        // This is a compute intensive task, we need to release the Gil
+        py.allow_threads(move || {
+            let bytes =
+                self.0.as_bytes().map_err(PyIcechunkStoreError::RepositoryError)?;
+            Ok(Cow::Owned(bytes))
+        })
+    }
+
     #[staticmethod]
     fn fetch_config(
         py: Python<'_>,

icechunk-0.2.3/icechunk-python/tests/test_debuginfo.py

@@ -0,0 +1,6 @@
+from icechunk import print_debug_info
+
+
+def test_debug_info() -> None:
+    # simple test that this does not cause an error.
+    print_debug_info()

{icechunk-0.2.2 → icechunk-0.2.3}/icechunk-python/tests/test_pickle.py

@@ -25,6 +25,18 @@ def tmp_repo(tmpdir: Path) -> Repository:
     return repo
 
 
+def test_pickle_repository(tmpdir: Path, tmp_repo: Repository) -> None:
+    pickled = pickle.dumps(tmp_repo)
+    roundtripped = pickle.loads(pickled)
+    assert tmp_repo.list_branches() == roundtripped.list_branches()
+
+    storage = tmp_repo.storage
+    assert (
+        repr(storage)
+        == f"ObjectStorage(backend=LocalFileSystemObjectStoreBackend(path={tmpdir}))"
+    )
+
+
 def test_pickle_read_only(tmp_repo: Repository) -> None:
     tmp_session = tmp_repo.writable_session(branch="main")
     tmp_store = tmp_session.store

{icechunk-0.2.2 → icechunk-0.2.3}/pyproject.toml

@@ -19,7 +19,7 @@ license = "Apache-2.0"
 dynamic = ["version"]
 authors = [{ name = "Earthmover", email = "info@earthmover.io" }]
 
-dependencies = ["zarr>=3"]
+dependencies = ["zarr>=3,!=3.0.3"]
 
 [project.optional-dependencies]
 test = [

{icechunk-0.2.2/icechunk-python → icechunk-0.2.3}/python/icechunk/__init__.py

@@ -52,6 +52,7 @@ from icechunk.credentials import (
     containers_credentials,
     gcs_credentials,
     gcs_from_env_credentials,
+    gcs_refreshable_credentials,
     gcs_static_credentials,
     s3_anonymous_credentials,
     s3_credentials,
@@ -128,11 +129,13 @@ __all__ = [
     "containers_credentials",
     "gcs_credentials",
     "gcs_from_env_credentials",
+    "gcs_refreshable_credentials",
     "gcs_static_credentials",
     "gcs_storage",
     "in_memory_storage",
     "initialize_logs",
     "local_filesystem_storage",
+    "print_debug_info",
     "s3_anonymous_credentials",
     "s3_credentials",
     "s3_from_env_credentials",
@@ -144,4 +147,19 @@ __all__ = [
     "tigris_storage",
 ]
 
+
+def print_debug_info() -> None:
+    import platform
+    from importlib import import_module
+
+    print(f"platform:  {platform.platform()}")
+    print(f"python:  {platform.python_version()}")
+    print(f"icechunk:  {__version__}")
+    for package in ["zarr", "numcodecs", "xarray", "virtualizarr"]:
+        try:
+            print(f"{package}:  {import_module(package).__version__}")
+        except ModuleNotFoundError:
+            continue
+
+
 initialize_logs()

{icechunk-0.2.2/icechunk-python → icechunk-0.2.3}/python/icechunk/_icechunk_python.pyi

@@ -947,6 +947,9 @@ class PyRepository:
     ) -> PyRepository: ...
     @staticmethod
     def exists(storage: Storage) -> bool: ...
+    @classmethod
+    def from_bytes(cls, data: bytes) -> PyRepository: ...
+    def as_bytes(self) -> bytes: ...
     @staticmethod
     def fetch_config(storage: Storage) -> RepositoryConfig | None: ...
     def save_config(self) -> None: ...
@@ -1418,6 +1421,7 @@ class Storage:
         *,
         config: dict[str, str] | None = None,
     ) -> Storage: ...
+    def __repr__(self) -> str: ...
     def default_settings(self) -> StorageSettings: ...
 
 class VersionSelection(Enum):

{icechunk-0.2.2 → icechunk-0.2.3}/python/icechunk/repository.py

@@ -140,6 +140,16 @@ class Repository:
         """
         return PyRepository.exists(storage)
 
+    def __getstate__(self) -> object:
+        return {
+            "_repository": self._repository.as_bytes(),
+        }
+
+    def __setstate__(self, state: object) -> None:
+        if not isinstance(state, dict):
+            raise ValueError("Invalid repository state")
+        self._repository = PyRepository.from_bytes(state["_repository"])
+
     @staticmethod
     def fetch_config(storage: Storage) -> RepositoryConfig | None:
         """