bucketbag 0.4.0__tar.gz

bucketbag-0.4.0/.gitignore

@@ -0,0 +1,12 @@
+__pycache__/
+*.py[cod]
+.venv/
+.venv-*/
+venv/
+*.egg-info/
+dist/
+build/
+.ruff_cache/
+.pytest_cache/
+.DS_Store
+*.parquet

bucketbag-0.4.0/CHANGELOG.md

@@ -0,0 +1,76 @@
+# Changelog
+
+## 0.4.0 — 2026-11-03
+
+A rewrite of the internals around a written contract (CONTRACT.md), plus a run report for
+agents. First release on PyPI: `pip install bucketbag`.
+
+### Breaking changes and how to migrate
+
+- **`Bag` resume needs a done-suffix.** Write `to_bucket(out, done=".md")` when each input `k`
+  writes `k + ".md"`. An input is done only when that exact output exists, so `pages/p10.md`
+  no longer marks `pages/p1` done. `compute(resume=True)` without `done=` raises; pass
+  `resume=False` to process everything.
+- **The map function is called as `fn(items)`, or `fn(items, ctx)` with `setup=`.** bucketbag
+  no longer inspects its signature, so a two-parameter `fn` without `setup=` now gets one
+  argument (before, it got `None` as the second).
+- **`prefix=` joins below a prefix inside the bucket reference**, as a directory:
+  `iter_keys("ns/b/out", prefix="part-")` lists `out/part-*` (before, `prefix=` replaced it).
+  `prefix=""` adds nothing; before, it listed the whole bucket.
+- **A trailing `/` in the bucket reference is kept**: `"ns/b/a/b/"` lists the directory
+  `a/b` only (before, it also matched `a/bc/`).
+- **Globs follow fsspec's rules** (`HfFileSystem.glob`): `[...]` is a character class, `*` no
+  longer matches an empty path segment, and `**` inside a name (`**.jp2`) raises; write
+  `**/*.jp2`.
+- **Settings that cannot work raise before anything is listed**: neither `n`/`batch_size` nor
+  `max_bytes` set, a value below 1, a negative `prefetch`, a bool, nan or inf.
+- **`completed_keys` raises when a readable shard lacks `column=`** (before, every read error
+  became a warning, so a wrong column silently re-ran everything).
+- **`batched_files(max_workers=)` is removed**; `prefetch` sets the number of download threads.
+- **`LoadedItem` is frozen.** Assigning to `item.key` or `item.path` raises
+  `FrozenInstanceError`; build a new `LoadedItem(key=..., path=...)` instead.
+- **Dependency floors:** `huggingface-hub>=1.21`, `fsspec>=2023.12`, `hf-xet>=1.5.2`.
+
+### New
+
+- Hub web URLs work as bucket references (`https://huggingface.co/buckets/ns/b/tree/x/`).
+- A run report: `BagStats` splits the time into download wait, map and upload; `bound_by`
+  names the slowest stage and `hint` the setting to change. `compute()` logs its settings,
+  a progress line after batches (at most every 30 s) and the report. bucketbag's log lines
+  print to stderr as `[bucketbag] …` until your script configures logging.
+- A warning before the first download when the largest `prefetch + 1` batches would not fit in
+  the scratch dir (on `/dev/shm`, running out kills the process).
+- `Bag.compute()` and `take()` warn once when an input wrote no `k + done` output (it would
+  rerun forever).
+- `Bag.compute()` with nothing left to do returns without running `setup()` (no model load).
+- `py.typed`: the package is fully typed.
+
+### Fixed
+
+- `batched_files` kept `prefetch + 2` batches on disk; it now keeps at most `prefetch + 1`.
+- Temp files no longer leak when Ctrl-C arrives during a download wait, or when a queued
+  download stored a `BaseException`.
+- `Bag.take()` honours `max_bytes`.
+
+### Internal
+
+- A pure core (`_core`) inside a thin I/O shell (`_hub`); one owner per temp dir; one run loop
+  for `take()` and `compute()`.
+- CONTRACT.md states every rule; `tests/test_contract.py` checks each one against an in-memory
+  Hub that binds every call to the real `HfApi` signatures, plus Hypothesis fault fuzzing.
+- CI: ruff format, ty on src and tests, a complexity limit, and a 15-minute job timeout.
+
+## 0.3.1 — 2026-08-26
+
+- Fix: `prefix` over-matched sibling directories in `iter_keys`, `batched_files` and
+  `completed_keys`. `prefix="a/b/"` returned `a/bc/y` too. Two causes: bucketbag stripped the
+  trailing slash before calling the Hub, and the Hub's pagination `Link: next` URL carries a
+  literal `/` that a 302 removes, so pages after the first of a recursive listing match on the
+  slash-less string prefix. The slash is now sent verbatim and keys are re-filtered
+  client-side with the caller's original prefix: a trailing `/` means "this directory"; no
+  slash keeps string-prefix semantics.
+
+## 0.3.0
+
+- `max_bytes` raises (not warns) when it cannot be honored (string keys carry no size).
+- See git tags `v0.3.0` / `v0.2.0` for earlier release notes.

bucketbag-0.4.0/CONTRACT.md

@@ -0,0 +1,108 @@
+# bucketbag contract
+
+What bucketbag promises. `tests/test_contract.py` checks each rule against an in-memory Hub
+(`tests/fake_bucket.py`) that follows the Hub's own listing rules. A rule marked **gap** is one
+the code breaks; its test is a strict xfail until the fix lands. No gap is open.
+
+## 1. Addressing
+
+- A bucket reference is `ns/bucket[/prefix]`, `hf://buckets/ns/bucket[/prefix]` or a Hub web
+  URL (`https://huggingface.co/buckets/ns/bucket/tree/prefix`). Anything else raises
+  `ValueError`, including an `hf://` URI of a model or dataset.
+- A prefix is a string prefix of the key, as on the Hub. A trailing `/` limits it to one
+  directory (`a/b/` lists `a/b/x`, not `a/bc/y`); without the slash, siblings match too.
+  The same rule holds for `prefix=` and for a prefix inside the bucket reference.
+- `prefix=` joins below a prefix inside the reference, as directories: `ns/b/out` with
+  `prefix="part-"` lists `out/part-*`. `prefix=""` or `None` adds nothing.
+- Globs (`include`, `exclude`) match the whole key from the bucket root, with fsspec's rules,
+  the ones `HfFileSystem.glob` uses: `*` stays inside one path segment, `**` as a whole segment
+  crosses `/`, `?` is one character and `[...]` a character class. `**` inside a name
+  (`**.jp2`) raises `ValueError`; write `**/*.jp2`. (`hf buckets sync --include` uses
+  `fnmatch`, where `*` also crosses `/`.)
+- Listings return full keys from the bucket root, sorted, files only. `start_after` skips
+  keys up to and including it. `limit` keeps the first `limit` keys in the Hub's listing
+  order, which is lexical today but not documented (issue #9).
+- `put_files(pairs, "ns/bucket/prefix")` writes each key under that prefix, joined as a
+  directory.
+
+## 2. Temporary files
+
+`batched_files` deletes a batch's files when the loop moves past it. It also deletes them when
+the loop body raises, when the caller stops early, and for batches downloaded ahead
+(`prefetch`). `tests/test_batched_files.py` covers each `prefetch` level.
+
+At most `prefetch + 1` batches are on disk at once: the one you are processing and `prefetch`
+downloading ahead. `prefetch` is also the number of download threads (at least one). Downloads
+always run in a thread, so Ctrl-C waits for the download in flight, then deletes its files.
+A second Ctrl-C during that wait stops the clean-up, and the batches still downloading may
+stay on disk.
+
+"Stops early" means the generator is closed. A `for` loop over `batched_files(...)` closes it
+when the loop ends. A caller that keeps the generator in a variable must call `.close()`
+(or use `contextlib.closing`); until then the current and lookahead batches stay on disk.
+
+## 3. Resume (`Bag`)
+
+- `to_bucket(out, done=".md")` names the output that marks an input done: an input `k` is done
+  when `k + ".md"` exists under `out` (`pages/p1.jp2` → `pages/p1.jp2.md`). The lookup is exact,
+  so no other output can mark `k` done (`pages/p10.md` never marks `pages/p1`). `compute()`
+  processes only inputs that are not done; `compute(resume=False)` processes all of them.
+  With nothing to do, `compute()` returns without running `setup()`.
+- `compute(resume=True)` without `done=` raises `ValueError` before anything is listed.
+- An input whose batch wrote no `k + done` output is logged as a warning, once per `compute()`
+  or `take()`. Resume cannot mark it done, so it runs again on every re-run.
+- `take(n)` runs the plan on the first `n` inputs and writes nothing. `setup()` runs at most
+  once per `compute()` or `take()`.
+
+## 4. Failures (`Bag.compute`)
+
+| What fails | What happens |
+|---|---|
+| The map function raises | `compute()` raises that error. Earlier batches' outputs stay. |
+| A download fails | `compute()` raises that error. Earlier batches' outputs stay. |
+| An upload fails | `compute()` raises that error. Outputs that landed before it count as done.* |
+
+In every row the temporary files are deleted, and a re-run processes exactly the inputs whose
+`done` output is missing. Hypothesis checks this over random mixes of failures (`tests/test_fault_fuzz.py`).
+
+\* The server applies the files of an upload that succeed and reports the rest.
+huggingface_hub 2.x raises `BucketBatchError` on that report. 1.x checked only the HTTP status,
+so `compute()` went on, and the missing outputs were redone on the next run.
+
+A file that is listed but gone by download time: with string keys
+(`batched_files(keys=[...])`) the hub skips it with a warning, and bucketbag leaves it out of
+the batch. `Bag` passes `BucketFile` objects, which the hub downloads by content hash without
+an existence check. What happens then is not verified live; the hub source suggests the download
+raises (`EntryNotFoundError`) at least when the gone file comes first in the batch.
+
+## 5. Arguments
+
+- `token=` reaches every hub call.
+- `batch_size` caps a batch's file count and `max_bytes` its total size; a single file larger
+  than `max_bytes` forms its own batch. `max_bytes` with string keys raises `ValueError`:
+  string keys carry no size.
+- Settings that cannot work raise before anything is listed. `n` (`batch_size` in `Bag`) and
+  `prefetch` must be whole numbers; `max_bytes` may be any finite number, such as `4e9`. A bool,
+  nan or inf raises `TypeError`. At least one of `n` and `max_bytes` must be set, each at least
+  1, and `prefetch` must be 0 or more (`ValueError`). `Bag.map_batches` checks its bounds when
+  the plan is built.
+- The map function is called as `fn(items)`, or `fn(items, ctx)` when `setup=` is given. Its
+  signature is not inspected.
+- `completed_keys` skips a shard it cannot read, with a warning. A readable shard without
+  `column` raises `ValueError`, so a wrong `column=` cannot make resume rerun everything.
+
+## 6. Run report (`Bag`)
+
+- `compute()` returns `BagStats` with the time spent waiting for the next batch (`wait_s`: its
+  download, plus deleting the last batch's files), in the map function (`map_s`) and writing
+  outputs (`upload_s`). The three exclude `setup()` and
+  listing; `seconds` includes them. `bound_by` names the largest of the three and `hint` the
+  setting to change. When no batch ran, `bound_by` is `None` and `hint` is `""`.
+  `print(stats)` shows the split and the verdict.
+- `compute()` logs its settings when it starts and its `BagStats` when it ends. `compute()` and
+  `take()` log a progress line after a batch, at most every 30 seconds: inputs done out of the
+  total, the rate, the time left and the split. A long batch delays its line. All go to the `bucketbag` logger at INFO, and print to stderr as
+  `[bucketbag] …` lines until the caller configures logging; then the caller's handlers get
+  them.
+- `batched_files` warns before the first download when the largest `prefetch + 1` batches
+  would not fit in the scratch dir's free space. String keys carry no size, so they never warn.

bucketbag-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Daniel van Strien
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bucketbag-0.4.0/PKG-INFO

@@ -0,0 +1,180 @@
+Metadata-Version: 2.5
+Name: bucketbag
+Version: 0.4.0
+Summary: A minimal, toolz-style helper for batched, bounded-memory access to Hugging Face bucket files.
+Project-URL: Homepage, https://github.com/davanstrien/bucketbag
+Project-URL: Repository, https://github.com/davanstrien/bucketbag
+Author: Daniel van Strien
+License-Expression: MIT
+License-File: LICENSE
+Keywords: batch,buckets,dataset,huggingface,toolz
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: fsspec>=2023.12
+Requires-Dist: hf-xet>=1.5.2
+Requires-Dist: huggingface-hub>=1.21
+Requires-Dist: pillow
+Requires-Dist: toolz
+Description-Content-Type: text/markdown
+
+# bucketbag
+
+**Experimental.** A small, [toolz](https://toolz.readthedocs.io/)-style library for batch-processing
+**large collections of files in [Hugging Face buckets](https://huggingface.co/docs/hub/en/storage-buckets)** — an experiment in what bucket-scale batch work
+wants as an interface:
+
+- **Bounded scratch** — download a batch, process, delete, repeat. Peak footprint ≈ `(prefetch + 1) × max_bytes`, never the corpus.
+- **Resume** — re-runs skip work already done (long [HF Jobs](https://huggingface.co/docs/hub/en/jobs) can stop and restart).
+- **Zero cleanup code** — temp files removed as the loop advances, even on exceptions.
+- **Throughput knobs** — prefetch overlaps download with compute; `boost()` raises xet's small-file concurrency.
+
+Mounting a bucket in a Job (`hf jobs run -v hf://buckets/...`) is the simplest way to use one; bucketbag explores the
+**explicit API path** for batch workloads where footprint, resume, and concurrency should be under
+your control. Not faster than the raw API (same transport), and not a framework.
+
+## Install
+
+```toml
+# /// script
+# dependencies = ["bucketbag==0.4.0"]
+# ///
+```
+
+Or `pip install bucketbag`. Pin a version: bucketbag is experimental and 0.x releases break.
+[CHANGELOG.md](https://github.com/davanstrien/bucketbag/blob/main/CHANGELOG.md) says how to move from 0.3.
+
+## The read loop
+
+```python
+from bucketbag import batched_files
+
+for batch in batched_files("ns/my-bucket", include="**/*.jp2", max_bytes=4 * 2**30):
+    for it in batch:                 # LoadedItem, already on local disk
+        work(it.path)                # or it.bytes / it.image / it.text() / it.json()
+    # ↑ this batch's files are deleted as the loop advances
+```
+
+Default scratch dir is `/dev/shm` (**RAM tmpfs, not disk**) — size `(prefetch + 1) × max_bytes`
+against memory, or pass `dir=`. One rule: don't keep a `LoadedItem`/`.path` past its batch.
+
+## Writing + resume
+
+```python
+from bucketbag import iter_keys, batched_files, put_files
+
+done = {k.removesuffix(".md") for k in iter_keys(OUT, include="**/*.md")}
+keys = [k for k in iter_keys(SRC, include="**/*.jp2", objects=True) if k.path not in done]
+for batch in batched_files(SRC, keys=keys, max_bytes=4 * 2**30):
+    put_files([(it.key + ".md", ocr(it)) for it in batch], OUT)   # one API call per batch
+```
+
+`iter_keys(..., objects=True)` yields `BucketFile` objects (with `.size`) instead of key
+strings — pass those to `batched_files(keys=…, max_bytes=…)` so the byte bound can be honored.
+Bare string keys carry no size, so **`max_bytes` + string keys raises `ValueError`** (fail fast
+rather than silently run unbounded against the default RAM-tmpfs scratch — an OOMKill, not a
+disk-full). Let `batched_files` do the listing (omit `keys=`) and it keeps the sizes for you.
+
+Parquet outputs are your tool's job (polars / pyarrow straight to the bucket);
+`completed_keys(OUT)` reads a done-set back from a `__source_key` column (pyarrow in *your* deps).
+
+## `Bag` — the loop as a lazy plan (experimental)
+
+```python
+from bucketbag import Bag
+
+bag = (Bag.from_bucket(SRC, include="**/*.jp2")
+         .map_batches(ocr_batch, batch_size=32, setup=load_model)   # fn(items[, ctx]) -> [(key, content)]
+         .to_bucket(OUT, done=".md"))                                  # input k is done once k + ".md" exists
+bag.take(3)        # dev peek: outputs only, nothing written
+stats = bag.compute()  # list -> skip done -> download -> fn -> put_files; kill + re-run safe
+print(stats)  # for example:
+# 476 processed (+24 skipped as done) in 15 batches -> 952 outputs, 3712.4s
+# time: download wait 1%, map 98%, upload 1%; bound by map. The map function is the limit: ...
+```
+
+`stats.bound_by` (`"download"`, `"map"` or `"upload"`) and `stats.hint` tell a person or an
+agent what to tune next. `compute()` also logs its settings, a progress line after batches
+(at most every 30 s) and the same report at the end. They print to stderr as `[bucketbag] …`
+lines, so they show up in `hf jobs logs`, until your script configures logging; then your
+handlers get them.
+
+Resume: `done=".md"` names the output that marks an input done, and the lookup is exact
+(`pages/p10.md` never marks `pages/p1` done). `compute()` runs in-process today; fanning the same
+plan across HF Jobs (one shard per job) is the next layer.
+
+To follow a bucket that another job is still filling, re-run the plan. Each pass lists both
+buckets again and skips what is done; `setup()` runs only when a pass has work:
+
+```python
+import time
+
+while producer_running():
+    bag.compute()
+    time.sleep(60)
+bag.compute()  # once more: files may have landed during the last sleep
+```
+
+[CONTRACT.md](https://github.com/davanstrien/bucketbag/blob/main/CONTRACT.md) states what bucketbag promises; the tests check each rule.
+
+## API
+
+| | |
+| --- | --- |
+| `batched_files(bucket, *, keys, include, exclude, n=20, max_bytes, dir, prefetch=2, ...)` | download batches → yield `list[LoadedItem]` → auto-delete |
+| `iter_keys(bucket, *, prefix, include, exclude, start_after, limit, token)` | list + glob-filter + sort keys (no download) |
+| `put_files(pairs, out_bucket)` / `put_bytes` / `put_text` | write raw objects; `put_files` batches many per call |
+| `completed_keys(out_bucket, *, prefix, column="__source_key")` | done-set from parquet outputs |
+| `boost(*, file_concurrency=32)` | raise xet download concurrency (~2.5× on small files) |
+| `LoadedItem` | `.key` `.path` + lazy `.bytes` `.image` `.text()` `.json()` |
+| `Bag.from_bucket(...).map_batches(fn).to_bucket(out, done=".md")` + `.take(n)` / `.compute()` | the loop as a lazy plan |
+
+`bucket` = `"ns/bucket"`, `"ns/bucket/prefix"`, `"hf://buckets/ns/bucket/prefix"`, or a Hub web URL.
+`prefix=` joins below a prefix inside `bucket`, as a directory.
+Globs follow fsspec's rules (as in `HfFileSystem.glob`): `*` within a path segment, `**` across `/`.
+`prefix` is a string prefix and the trailing slash matters: `prefix="a/b/"` lists the
+directory `a/b` only; `prefix="a/b"` also matches siblings like `a/bc/y`. The prefix is sent
+verbatim to the Hub. Bucketbag also checks each returned key against it as a cheap safeguard
+against over-matching listings, including pagination regressions.
+
+## Performance
+
+Cold + disjoint, replicated (`examples/bench.py`, l4x1, ~0.8 MB jp2; ±10–20%):
+
+| | default xet | + `boost()` |
+| --- | --- | --- |
+| `bucketbag` (prefetch 2–4) | ~85–110 img/s | **~200–270 img/s** |
+| raw `download_bucket_files` | ~85–105 | ~210–260 |
+| `HfFileSystem` (32 threads) | ~90–110 | ~110 (bypasses xet) |
+| FUSE mount (same workload) | ~22 | — |
+
+Competitive with the raw API by design; the win is the interface. The one real throughput lever is
+xet's concurrent-file cap (default 8) — `boost()` raises it, ~2.5× on small files. Skip it for
+large files (over-subscription).
+
+### Decode, not transport
+
+If a pipeline is slow, time the download and the decode separately before tuning transport
+(`examples/decode_probe.py` does this). The transport is usually not the gate: on ~0.7 MB page
+images, `batched_files` moves 40 files in ~1 s, and decoding those 40 takes ~70 s. Three things
+to check in the decode step:
+
+- **Threads may not help.** Some Pillow decoders hold the GIL (JPEG 2000 does): 8 threads gave the
+  same speed as 1. A process pool did scale.
+- **Decode smaller.** Formats with resolution levels (JPEG 2000: `im.reduce`; JPEG: `im.draft()`)
+  can decode at 1/4 or 1/8 the pixels for a fraction of the time — usually all a classifier needs.
+- **Check the reduced size loads.** Pillow's JPEG 2000 `reduce>=2` fails with "broken data stream"
+  when its size rounding disagrees with OpenJPEG's; `safe_reduce()` in the example picks the
+  largest level that loads.
+
+## Not in scope
+
+DAGs, shuffles, custom formats, return values to a driver. Stages couple through bucket keys; a
+global reduce is a one-off polars job. Jobs fan-out is planned as a thin layer on `Bag`.

bucketbag-0.4.0/README.md

@@ -0,0 +1,153 @@
+# bucketbag
+
+**Experimental.** A small, [toolz](https://toolz.readthedocs.io/)-style library for batch-processing
+**large collections of files in [Hugging Face buckets](https://huggingface.co/docs/hub/en/storage-buckets)** — an experiment in what bucket-scale batch work
+wants as an interface:
+
+- **Bounded scratch** — download a batch, process, delete, repeat. Peak footprint ≈ `(prefetch + 1) × max_bytes`, never the corpus.
+- **Resume** — re-runs skip work already done (long [HF Jobs](https://huggingface.co/docs/hub/en/jobs) can stop and restart).
+- **Zero cleanup code** — temp files removed as the loop advances, even on exceptions.
+- **Throughput knobs** — prefetch overlaps download with compute; `boost()` raises xet's small-file concurrency.
+
+Mounting a bucket in a Job (`hf jobs run -v hf://buckets/...`) is the simplest way to use one; bucketbag explores the
+**explicit API path** for batch workloads where footprint, resume, and concurrency should be under
+your control. Not faster than the raw API (same transport), and not a framework.
+
+## Install
+
+```toml
+# /// script
+# dependencies = ["bucketbag==0.4.0"]
+# ///
+```
+
+Or `pip install bucketbag`. Pin a version: bucketbag is experimental and 0.x releases break.
+[CHANGELOG.md](https://github.com/davanstrien/bucketbag/blob/main/CHANGELOG.md) says how to move from 0.3.
+
+## The read loop
+
+```python
+from bucketbag import batched_files
+
+for batch in batched_files("ns/my-bucket", include="**/*.jp2", max_bytes=4 * 2**30):
+    for it in batch:                 # LoadedItem, already on local disk
+        work(it.path)                # or it.bytes / it.image / it.text() / it.json()
+    # ↑ this batch's files are deleted as the loop advances
+```
+
+Default scratch dir is `/dev/shm` (**RAM tmpfs, not disk**) — size `(prefetch + 1) × max_bytes`
+against memory, or pass `dir=`. One rule: don't keep a `LoadedItem`/`.path` past its batch.
+
+## Writing + resume
+
+```python
+from bucketbag import iter_keys, batched_files, put_files
+
+done = {k.removesuffix(".md") for k in iter_keys(OUT, include="**/*.md")}
+keys = [k for k in iter_keys(SRC, include="**/*.jp2", objects=True) if k.path not in done]
+for batch in batched_files(SRC, keys=keys, max_bytes=4 * 2**30):
+    put_files([(it.key + ".md", ocr(it)) for it in batch], OUT)   # one API call per batch
+```
+
+`iter_keys(..., objects=True)` yields `BucketFile` objects (with `.size`) instead of key
+strings — pass those to `batched_files(keys=…, max_bytes=…)` so the byte bound can be honored.
+Bare string keys carry no size, so **`max_bytes` + string keys raises `ValueError`** (fail fast
+rather than silently run unbounded against the default RAM-tmpfs scratch — an OOMKill, not a
+disk-full). Let `batched_files` do the listing (omit `keys=`) and it keeps the sizes for you.
+
+Parquet outputs are your tool's job (polars / pyarrow straight to the bucket);
+`completed_keys(OUT)` reads a done-set back from a `__source_key` column (pyarrow in *your* deps).
+
+## `Bag` — the loop as a lazy plan (experimental)
+
+```python
+from bucketbag import Bag
+
+bag = (Bag.from_bucket(SRC, include="**/*.jp2")
+         .map_batches(ocr_batch, batch_size=32, setup=load_model)   # fn(items[, ctx]) -> [(key, content)]
+         .to_bucket(OUT, done=".md"))                                  # input k is done once k + ".md" exists
+bag.take(3)        # dev peek: outputs only, nothing written
+stats = bag.compute()  # list -> skip done -> download -> fn -> put_files; kill + re-run safe
+print(stats)  # for example:
+# 476 processed (+24 skipped as done) in 15 batches -> 952 outputs, 3712.4s
+# time: download wait 1%, map 98%, upload 1%; bound by map. The map function is the limit: ...
+```
+
+`stats.bound_by` (`"download"`, `"map"` or `"upload"`) and `stats.hint` tell a person or an
+agent what to tune next. `compute()` also logs its settings, a progress line after batches
+(at most every 30 s) and the same report at the end. They print to stderr as `[bucketbag] …`
+lines, so they show up in `hf jobs logs`, until your script configures logging; then your
+handlers get them.
+
+Resume: `done=".md"` names the output that marks an input done, and the lookup is exact
+(`pages/p10.md` never marks `pages/p1` done). `compute()` runs in-process today; fanning the same
+plan across HF Jobs (one shard per job) is the next layer.
+
+To follow a bucket that another job is still filling, re-run the plan. Each pass lists both
+buckets again and skips what is done; `setup()` runs only when a pass has work:
+
+```python
+import time
+
+while producer_running():
+    bag.compute()
+    time.sleep(60)
+bag.compute()  # once more: files may have landed during the last sleep
+```
+
+[CONTRACT.md](https://github.com/davanstrien/bucketbag/blob/main/CONTRACT.md) states what bucketbag promises; the tests check each rule.
+
+## API
+
+| | |
+| --- | --- |
+| `batched_files(bucket, *, keys, include, exclude, n=20, max_bytes, dir, prefetch=2, ...)` | download batches → yield `list[LoadedItem]` → auto-delete |
+| `iter_keys(bucket, *, prefix, include, exclude, start_after, limit, token)` | list + glob-filter + sort keys (no download) |
+| `put_files(pairs, out_bucket)` / `put_bytes` / `put_text` | write raw objects; `put_files` batches many per call |
+| `completed_keys(out_bucket, *, prefix, column="__source_key")` | done-set from parquet outputs |
+| `boost(*, file_concurrency=32)` | raise xet download concurrency (~2.5× on small files) |
+| `LoadedItem` | `.key` `.path` + lazy `.bytes` `.image` `.text()` `.json()` |
+| `Bag.from_bucket(...).map_batches(fn).to_bucket(out, done=".md")` + `.take(n)` / `.compute()` | the loop as a lazy plan |
+
+`bucket` = `"ns/bucket"`, `"ns/bucket/prefix"`, `"hf://buckets/ns/bucket/prefix"`, or a Hub web URL.
+`prefix=` joins below a prefix inside `bucket`, as a directory.
+Globs follow fsspec's rules (as in `HfFileSystem.glob`): `*` within a path segment, `**` across `/`.
+`prefix` is a string prefix and the trailing slash matters: `prefix="a/b/"` lists the
+directory `a/b` only; `prefix="a/b"` also matches siblings like `a/bc/y`. The prefix is sent
+verbatim to the Hub. Bucketbag also checks each returned key against it as a cheap safeguard
+against over-matching listings, including pagination regressions.
+
+## Performance
+
+Cold + disjoint, replicated (`examples/bench.py`, l4x1, ~0.8 MB jp2; ±10–20%):
+
+| | default xet | + `boost()` |
+| --- | --- | --- |
+| `bucketbag` (prefetch 2–4) | ~85–110 img/s | **~200–270 img/s** |
+| raw `download_bucket_files` | ~85–105 | ~210–260 |
+| `HfFileSystem` (32 threads) | ~90–110 | ~110 (bypasses xet) |
+| FUSE mount (same workload) | ~22 | — |
+
+Competitive with the raw API by design; the win is the interface. The one real throughput lever is
+xet's concurrent-file cap (default 8) — `boost()` raises it, ~2.5× on small files. Skip it for
+large files (over-subscription).
+
+### Decode, not transport
+
+If a pipeline is slow, time the download and the decode separately before tuning transport
+(`examples/decode_probe.py` does this). The transport is usually not the gate: on ~0.7 MB page
+images, `batched_files` moves 40 files in ~1 s, and decoding those 40 takes ~70 s. Three things
+to check in the decode step:
+
+- **Threads may not help.** Some Pillow decoders hold the GIL (JPEG 2000 does): 8 threads gave the
+  same speed as 1. A process pool did scale.
+- **Decode smaller.** Formats with resolution levels (JPEG 2000: `im.reduce`; JPEG: `im.draft()`)
+  can decode at 1/4 or 1/8 the pixels for a fraction of the time — usually all a classifier needs.
+- **Check the reduced size loads.** Pillow's JPEG 2000 `reduce>=2` fails with "broken data stream"
+  when its size rounding disagrees with OpenJPEG's; `safe_reduce()` in the example picks the
+  largest level that loads.
+
+## Not in scope
+
+DAGs, shuffles, custom formats, return values to a driver. Stages couple through bucket keys; a
+global reduce is a one-off polars job. Jobs fan-out is planned as a thin layer on `Bag`.

bucketbag-0.4.0/pyproject.toml

@@ -0,0 +1,107 @@
+[project]
+name = "bucketbag"
+version = "0.4.0"
+description = "A minimal, toolz-style helper for batched, bounded-memory access to Hugging Face bucket files."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Daniel van Strien" }]
+keywords = ["huggingface", "buckets", "batch", "toolz", "dataset"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Typing :: Typed",
+]
+dependencies = [
+    # 1.21: parse_hf_uri reads Hub web URLs (1.13 rejects them).
+    "huggingface-hub>=1.21",
+    "hf-xet>=1.5.2",
+    # 2023.12: fsspec.utils.glob_translate, the glob rules HfFileSystem.glob uses.
+    "fsspec>=2023.12",
+    "toolz",
+    "pillow",
+]
+
+[project.urls]
+Homepage = "https://github.com/davanstrien/bucketbag"
+Repository = "https://github.com/davanstrien/bucketbag"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/bucketbag"]
+
+[tool.hatch.build.targets.sdist]
+# Only what a source build and its tests need; never local venvs or experiment outputs.
+include = [
+    "/src",
+    "/tests",
+    "/README.md",
+    "/CHANGELOG.md",
+    "/CONTRACT.md",
+    "/LICENSE",
+    "/pyproject.toml",
+]
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+# ANN: the package and its tests are fully typed. C90/PLR09: a complexity ratchet; no function
+# may grow past these limits.
+select = ["E", "F", "I", "UP", "B", "ANN", "C90", "PLR0912", "PLR0915"]
+# Any is deliberate where the type is the caller's (LoadedItem.json(), setup() results).
+ignore = ["ANN401"]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 10
+
+[tool.ruff.lint.per-file-ignores]
+# Scripts, not library code: examples measure things, experiments vendor recipe code unchanged.
+"examples/**" = ["ANN", "C90", "PLR"]
+"experiments/**" = ["B905", "ANN", "C90", "PLR"]
+
+[tool.ty.src]
+include = ["src", "tests"]
+
+[tool.ty.analysis]
+# pyarrow is an optional import in completed_keys, not a bucketbag dependency.
+allowed-unresolved-imports = ["pyarrow.**"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = ["-ra", "--strict-markers", "--strict-config"]
+# A test stuck for 120 s dumps every thread's stack and ends the run. The thread method is
+# needed: with the signal method a stuck prefetch worker still blocks the process from exiting.
+timeout = 120
+timeout_method = "thread"
+
+[dependency-groups]
+dev = [
+    "hypothesis>=6.155.7",
+    "pytest>=9.1.1",
+    "pytest-timeout>=2.4",
+    "ruff>=0.15",
+    "ty>=0.0.91",
+]
+
+# Pin the index to public PyPI so the lockfile resolves against the canonical registry,
+# independent of any ambient uv config (e.g. a machine-local index-url). This keeps
+# `uv sync` reproducible for CI and other contributors.
+#
+# Same class of problem, second instance: regenerate the lock with `uv lock --no-config`.
+# A machine-global `exclude-newer` otherwise bakes an `[options]` block into uv.lock that
+# CI (with no such config) rejects under `--locked`. Locally, prefer
+# `uv sync --frozen` / `uv run --no-sync` to avoid churning the lock back.
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+default = true