icechunk 2.2.2__tar.gz → 2.3.0a1__tar.gz

{icechunk-2.2.2 → icechunk-2.3.0a1}/Cargo.lock

@@ -156,18 +156,6 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8"
 
-[[package]]
-name = "async-compression"
-version = "0.4.47"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef217a77a86a6e3dab9a5b3c81dc445b603fe743a90c1cb10a2f2144628d8cfa"
-dependencies = [
- "compression-codecs",
- "compression-core",
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "async-recursion"
 version = "1.1.1"
@@ -1055,23 +1043,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "compression-codecs"
-version = "0.4.42"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "257c7085cbb71be72d8fb97edff08b03d86d5d9f2222b9cc34a6bec87093bc10"
-dependencies = [
- "compression-core",
- "zstd",
- "zstd-safe",
-]
-
-[[package]]
-name = "compression-core"
-version = "0.4.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
-
 [[package]]
 name = "console"
 version = "0.16.3"
@@ -2279,11 +2250,10 @@ dependencies = [
 
 [[package]]
 name = "icechunk"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "anyhow",
  "assert_fs",
- "async-compression",
  "async-recursion",
  "async-stream",
  "async-trait",
@@ -2316,6 +2286,7 @@ dependencies = [
  "rand 0.10.2",
  "regex",
  "reqwest 0.13.5",
+ "rlimit",
  "rmp-serde",
  "rstest",
  "rstest_reuse",
@@ -2347,7 +2318,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-arrow-object-store"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -2374,7 +2345,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-format"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "base32",
  "bytes",
@@ -2425,7 +2396,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-macros"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "quote",
  "syn 3.0.5",
@@ -2433,7 +2404,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-python"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2460,7 +2431,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-s3"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -2487,7 +2458,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-storage"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -2506,7 +2477,7 @@ dependencies = [
 
 [[package]]
 name = "icechunk-types"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 dependencies = [
  "serde",
  "serde_with",
@@ -4321,6 +4292,15 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rlimit"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f35ee2729c56bb610f6dba436bf78135f728b7373bdffae2ec815b2d3eb98cc3"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "rmp"
 version = "0.8.15"

{icechunk-2.2.2 → icechunk-2.3.0a1}/Cargo.toml

@@ -9,18 +9,17 @@ readme = "README.md"
 
 [workspace.dependencies]
 # Internal crates
-icechunk = { path = "icechunk", version = "2.2.2" }
-icechunk-arrow-object-store = { path = "icechunk-arrow-object-store", version = "2.2.2", default-features = false }
-icechunk-format = { path = "icechunk-format", version = "2.2.2" }
-icechunk-macros = { path = "icechunk-macros", version = "2.2.2" }
-icechunk-s3 = { path = "icechunk-s3", version = "2.2.2" }
-icechunk-storage = { path = "icechunk-storage", version = "2.2.2" }
-icechunk-types = { path = "icechunk-types", version = "2.2.2" }
+icechunk = { path = "icechunk", version = "2.3.0-alpha.1" }
+icechunk-arrow-object-store = { path = "icechunk-arrow-object-store", version = "2.3.0-alpha.1", default-features = false }
+icechunk-format = { path = "icechunk-format", version = "2.3.0-alpha.1" }
+icechunk-macros = { path = "icechunk-macros", version = "2.3.0-alpha.1" }
+icechunk-s3 = { path = "icechunk-s3", version = "2.3.0-alpha.1" }
+icechunk-storage = { path = "icechunk-storage", version = "2.3.0-alpha.1" }
+icechunk-types = { path = "icechunk-types", version = "2.3.0-alpha.1" }
 
 # External dependencies
 anyhow = "1.0.102"
 assert_fs = "1.1.4"
-async-compression = { version = "0.4.47", features = ["tokio", "zstd"] }
 async-recursion = "1.1.1"
 async-stream = "0.3.6"
 async-trait = "0.1.89"
@@ -62,6 +61,7 @@ quote = "1.0"
 rand = "0.10.1"
 regex = "1.12.3"
 reqwest = "0.13"
+rlimit = "0.11.0"
 rmp-serde = "1.3.1"
 rstest = "0.27.0"
 rstest_reuse = "0.7.0"

{icechunk-2.2.2 → icechunk-2.3.0a1}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: icechunk
-Version: 2.2.2
+Version: 2.3.0a1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
 Requires-Dist: zarr>=3.1.0
 License-File: LICENSE
 Summary: Icechunk Python

{icechunk-2.2.2 → icechunk-2.3.0a1}/icechunk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "icechunk"
-version = "2.2.2"
+version = "2.3.0-alpha.1"
 description = "Transactional storage engine for Zarr designed for use on cloud object storage"
 readme = "README.md"
 repository = "https://github.com/earth-mover/icechunk"
@@ -22,7 +22,6 @@ icechunk-storage.workspace = true
 icechunk-types.workspace = true
 icechunk-s3 = { workspace = true, optional = true }
 
-async-compression.workspace = true
 async-recursion.workspace = true
 async-stream.workspace = true
 async-trait.workspace = true
@@ -81,6 +80,9 @@ tokio = { workspace = true, features = [
   "io-util",
 ] }
 
+[target.'cfg(unix)'.dependencies]
+rlimit.workspace = true
+
 [dev-dependencies]
 icechunk-macros.workspace = true
 # test-util exposes write-id injection for the lost-response regression tests.
@@ -144,6 +146,12 @@ name = "icechunk"
 path = "src/bin/icechunk/main.rs"
 required-features = ["cli"]
 
+# the report layer needs tracing_subscriber, which only `logs` pulls in
+[[example]]
+name = "gc_bench"
+path = "examples/gc_bench/main.rs"
+required-features = ["logs"]
+
 [[test]]
 name = "main"
 path = "tests/main.rs"

icechunk-2.3.0a1/icechunk/examples/gc_bench/README.md

@@ -0,0 +1,68 @@
+# gc_bench
+
+Benchmark tool for garbage collection, repo stats, and object listing.
+
+- `build`, `gc`, `stats` work against a synthetic repository on local RustFS
+  (`just contup` first). See `just gc-bench --help`.
+- `list` measures listing throughput only. It is the only subcommand that
+  accepts a real S3 target, because it is the only one that is strictly read
+  only: it issues list requests and nothing else. It never reads, writes or
+  deletes an object, and it never prints anything that identifies one.
+
+## Running `list` on an EC2 instance in the bucket's region
+
+```sh
+# on the instance: install rustup (stable), clone the repo at this commit, then
+cargo build --profile bench --features logs --example gc_bench
+
+# temporary credentials, e.g. from `aws sts assume-role` or the console
+export AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... AWS_SESSION_TOKEN=...
+
+# the bench profile writes to target/release, not target/bench
+B=./target/release/examples/gc_bench
+$B list --s3-bucket <bucket> --s3-prefix <prefix> --s3-region <region> --mode single --duration-secs 30
+$B list --s3-bucket <bucket> --s3-prefix <prefix> --s3-region <region> --mode asset-manager --streams 64 --duration-secs 30
+$B list --s3-bucket <bucket> --s3-prefix <prefix> --s3-region <region> --mode asset-manager --streams 128 --duration-secs 30
+```
+
+`asset-manager` is the production path, `AssetManager::list_*_with_concurrency`,
+the one GC uses. `single` is the store's per-stream baseline; the production
+rate should scale with `--streams` until the cores parsing listing responses
+are saturated.
+
+Credentials are resolved in this order, and the run prints which source it used,
+never the values:
+
+1. `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in the environment become
+   static credentials, with `AWS_SESSION_TOKEN` if it is set. This is how
+   temporary STS credentials are passed.
+2. Otherwise the AWS SDK's default chain (instance role, shared config profile),
+   reported as `credentials: FromEnv`.
+
+## Modes
+
+- `single`: one `list_objects` stream over `<kind>/`, driven by one task
+  through `MeteringStorage`.
+- `asset-manager`: `AssetManager::list_*_with_concurrency` with `--streams`
+  listing tasks, one prefix each, off a shared queue. These tasks list on the
+  raw storage, not through `MeteringStorage`: keys are counted in a task-local
+  counter that is published to a shared atomic once per 1000 keys, and a ticker
+  samples that atomic once a second for the timeline.
+
+The report ends with `available cores` and the process CPU time, so a run that
+is CPU bound on one core is visible as such.
+
+`--kind chunks|manifests|snapshots|transaction_logs` picks the prefix
+(`chunks` by default), `--duration-secs` bounds the run (30 by default) and
+`--max-keys` stops it early. On the duration limit the streams are dropped,
+abandoning in-flight requests.
+
+The report gives the keys-per-second timeline, min/median/max/mean over the
+seconds after the first (the first second includes request warm-up), total keys,
+elapsed time and the approximate page count (`keys/1000`).
+
+Against the local `large` dataset instead of S3:
+
+```sh
+just gc-bench list --name large --mode asset-manager --streams 64 --duration-secs 20
+```