atlas-python 0.16.3__tar.gz → 0.16.4__tar.gz

{atlas_python-0.16.3 → atlas_python-0.16.4}/Cargo.lock

@@ -71,7 +71,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-python"
-version = "0.16.3"
+version = "0.16.4"
 dependencies = [
  "atlas-rust",
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-rust"
-version = "0.16.3"
+version = "0.16.4"
 dependencies = [
  "array-format",
  "async-trait",

{atlas_python-0.16.3 → atlas_python-0.16.4}/Cargo.toml

@@ -5,7 +5,7 @@ members = ["atlas-python"]
 # Registry name on crates.io (`atlas` is taken). The library is still
 # imported as `atlas` — see `[lib]` below.
 name = "atlas-rust"
-version = "0.16.3"
+version = "0.16.4"
 edition = "2024"
 description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
 license = "Apache-2.0"

{atlas_python-0.16.3 → atlas_python-0.16.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-python
-Version: 0.16.3
+Version: 0.16.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3

{atlas_python-0.16.3 → atlas_python-0.16.4}/atlas-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "atlas-python"
-version = "0.16.3"
+version = "0.16.4"
 edition = "2021"
 readme = "README.md"
 

{atlas_python-0.16.3 → atlas_python-0.16.4}/atlas-python/docs/cli.md

@@ -25,12 +25,20 @@ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
 
 ## Logging
 
-`--log-file PATH` appends every error and warning to a file, with the reason:
+`--log-file PATH` appends to the file you name. There is no default location,
+and no log at all without the flag. The command prints the absolute path it
+opened, so it is never a guess:
+
+```text
+atlas: logging to /home/you/ingest.log
+```
+
+The file gets every error and warning, with the reason:
 
 ```bash
 $ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
 $ cat ingest.log
-2026-09-01 14:30:41 INFO    atlas.cli: atlas 0.16.3: create /data/nc ...
+2026-09-01 14:30:41 INFO    atlas.cli: atlas 0.16.4: create /data/nc ...
 2026-09-01 14:30:41 INFO    atlas.ops: ingesting 2 file(s) into /data/collection
 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
 2026-09-01 14:30:41 INFO    atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
@@ -66,11 +74,25 @@ A failure part-way leaves no collection, and not a partial one.
 | `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
 | `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
 | `--skip-errors` | Skip files that fail instead of abandoning the collection |
+| `-j`, `--workers N` | Stage N files at once. About 3x on a many-core machine. Ordinals do not move |
+| `--convert-calendar` | Turn a cftime axis into exact Gregorian timestamps, each keeping its instant |
 | `--no-decode-times` | Keep a time axis as raw numbers, for a calendar that decodes to cftime |
 | `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
 | `-q`, `--quiet` | Do not list a file as it lands |
 
-Progress goes to stderr, so a pipe still reads stdout.
+Progress goes to stderr, so a pipe still reads stdout. Each line counts the
+files and says how many remain:
+
+```text
+Writing /data/collection from 3 file(s)
+  [1/3] 2024-01.nc  (2 left)
+  [2/3] 2024-02.nc  (1 left)
+  [3/3] 2024-03.nc  (0 left)
+3 dataset(s) written to /data/collection
+```
+
+`-q` turns the per-file lines off. The same counter goes to `--log-file`, for
+a run nobody watched.
 
 ### Unsupported dtypes
 
@@ -110,7 +132,7 @@ size. `--open-chunks` picks another strategy:
 
 | Mode | Reads | Stored chunk shape |
 |---|---|---|
-| `auto` *(default)* | blocks sized to `--chunk-size` | those blocks |
+| `auto` *(default)* | whole when the file is small, else blocks sized to `--chunk-size` | those blocks |
 | `native` | the file's own chunk encoding | that encoding |
 | `none` | each variable whole | one full-shape chunk |
 | JSON dict | as given, per dimension | as given |

{atlas_python-0.16.3 → atlas_python-0.16.4}/atlas-python/docs/guides/creating.md

@@ -140,8 +140,10 @@ that stream to stderr. See [Installation](../installation.md).
 These are one decision. The blocks a file *reads* in are the chunks it
 *stores* in.
 
-Each file opens with dask chunking. `open_chunks="auto"` is the default. dask
-then sizes the blocks to `chunk_size`, which is 128 MiB. Three results follow:
+`open_chunks="auto"` is the default, and it picks a strategy per file. A file
+well inside the block budget opens whole. The dask graph costs about 40 ms
+per file, and a small variable lands as one chunk either way. A larger
+file streams through dask, so memory stays bounded. Three results follow:
 
 - A file far larger than memory streams block by block, and does not read
   whole.
@@ -158,13 +160,14 @@ atlas.create("/data/nc", dest, chunk_size="64MiB")
 ```
 
 `chunk_size` is about the memory ceiling per variable. Lower it on a small
-machine. Raise it for larger stored chunks.
+machine. Raise it for larger stored chunks. It also moves the threshold at
+which `"auto"` stops opening a file whole.
 
 ### How files are opened
 
 | `open_chunks` | Reads | Stored chunk shape |
 |---|---|---|
-| `"auto"` *(default)* | blocks sized to `chunk_size` | those blocks |
+| `"auto"` *(default)* | whole when small, else blocks sized to `chunk_size` | those blocks |
 | `"native"` | the file's own chunk encoding | that encoding |
 | `None` | each variable whole | one full-shape chunk |
 | `{"time": 100}` | as given, per dimension | as given |
@@ -206,6 +209,51 @@ arrays["temperature"]["chunk_shape"]
 atlas show dest 2024-01.nc | grep _ChunkShape
 ```
 
+### Speed on many small files
+
+A directory of small files spends most of its time per file, not per byte. On
+213 KiB profile files the default runs at about 30 files per second, or five
+minutes for ten thousand.
+
+**`--workers N` stages N files at once.** It is the largest single win:
+
+```bash
+atlas create /data/nc /data/collection --workers 4
+```
+
+```text
+workers=1 :  32 files/s   1.00x
+workers=2 :  84 files/s   2.65x
+workers=4 : 120 files/s   3.77x
+workers=8 : 118 files/s   3.72x   <- plateau
+```
+
+The commit is the costly half of an ingest, about two thirds of it. It is pure
+Rust, holds no lock until its append, and releases the GIL, so it overlaps.
+The reads stay on the calling thread, and they are what flattens the curve
+near four.
+
+The reads have to stay there. netCDF4 sits on HDF5, which is not thread safe,
+and xarray locks an array read but not a variable open. Two threads inside
+`open_dataset` crash the process.
+
+Nothing else changes. `add_dataset` runs in file order, so every ordinal
+matches a sequential build. The summary sorts back into file order too. Only
+`progress` reports in completion order.
+
+Two more settings matter when that is still too slow:
+
+- **`--open-chunks native`** forces dask on every file. That costs about twice
+  as long on a small file. Use it only when the file's own chunking is what
+  you want stored.
+- **`--codec none`** trades size for speed. It saved about 20 percent on the
+  same files.
+
+Past the plateau, run several `atlas create` commands at once, one per part of
+the tree. Each has its own writer, so nothing serialises between them. That
+reached 3.7x on eight processes where threads reached 2.9x. The cost is one
+collection per part.
+
 ### The writer's own memory
 
 Staging runs on local disk. `array-format` spills each compressed chunk to a
@@ -233,8 +281,15 @@ used.
 atlas.create("/data/nc", dest, progress=lambda name: print(name))
 ```
 
-The CLI does this by default, to stderr, so a pipe still reads stdout. `-q`
-turns it off.
+The CLI does this by default, to stderr, so a pipe still reads stdout. Each
+line counts the files and says how many remain:
+
+```text
+  [ 12/10000] 000043_CFPOINT_3593_V0.nc  (9988 left)
+```
+
+`-q` turns the per-file lines off. Pass `--log-file PATH` to keep the same
+counter in a file. The command prints the absolute path it opened.
 
 ## What can go wrong
 

{atlas_python-0.16.3 → atlas_python-0.16.4}/atlas-python/tests/test_cli.py

@@ -186,6 +186,37 @@ def test_create_descends_by_default_and_no_recursive_opts_out(capsys, tmp_path):
     assert code == 0, err
 
 
+def test_create_counts_progress_and_what_is_left(capsys, netcdf_dir, tmp_path):
+    code, _, err = run(capsys, "create", str(netcdf_dir), str(tmp_path / "c"))
+    assert code == 0
+    assert "from 3 file(s)" in err
+    assert "[1/3] 2024-01.nc  (2 left)" in err
+    assert "[3/3] 2024-03.nc  (0 left)" in err
+
+
+def test_log_file_says_where_it_writes(capsys, netcdf_dir, tmp_path):
+    log = tmp_path / "run.log"
+    code, _, err = run(
+        capsys, "create", str(netcdf_dir), str(tmp_path / "c"), "-q",
+        "--log-file", str(log),
+    )
+    assert code == 0
+    assert f"logging to {log}" in err
+    # The log carries the same counter, for a run nobody watched.
+    text = log.read_text()
+    assert "[1/3] wrote 2024-01.nc" in text
+    assert "[3/3] wrote 2024-03.nc" in text
+
+
+def test_workers_flag_builds_the_same_collection(capsys, netcdf_dir, tmp_path):
+    run(capsys, "create", str(netcdf_dir), str(tmp_path / "one"), "-q")
+    run(capsys, "create", str(netcdf_dir), str(tmp_path / "many"), "-q", "-j", "4")
+
+    code, a, _ = run(capsys, "ls", str(tmp_path / "one"))
+    code, b, _ = run(capsys, "ls", str(tmp_path / "many"))
+    assert a.split() == b.split() == ["2024-01.nc", "2024-02.nc", "2024-03.nc"]
+
+
 # ── ls ───────────────────────────────────────────────────────────────
 
 

{atlas_python-0.16.3 → atlas_python-0.16.4}/atlas-python/tests/test_ops.py

@@ -148,6 +148,105 @@ def test_every_codec_round_trips(netcdf_dir, tmp_path, codec):
     assert len(atlas.list_datasets(str(dest))) == 3
 
 
+# ── parallel staging ─────────────────────────────────────────────────
+
+
+@pytest.fixture
+def many_netcdf_files(tmp_path):
+    """Enough files that several workers actually overlap."""
+    d = tmp_path / "many"
+    d.mkdir()
+    for i in range(12):
+        make_dataset((i % 12) + 1).to_netcdf(d / f"m{i:03d}.nc")
+    return d
+
+
+@pytest.mark.parametrize("workers", [2, 4])
+def test_workers_keep_the_ordinals_of_a_sequential_build(
+    many_netcdf_files, tmp_path, workers
+):
+    """add_dataset runs in file order, so a worker cannot move an ordinal."""
+    one = tmp_path / "one"
+    many = tmp_path / f"many{workers}"
+    atlas.create(many_netcdf_files, str(one))
+    atlas.create(many_netcdf_files, str(many), workers=workers)
+
+    assert atlas.list_datasets(str(one)) == atlas.list_datasets(str(many))
+    for name in atlas.list_datasets(str(one)):
+        assert (
+            atlas.describe(str(one), name)["ordinal"]
+            == atlas.describe(str(many), name)["ordinal"]
+        )
+
+
+def test_workers_produce_the_same_collection(many_netcdf_files, tmp_path):
+    one = tmp_path / "one"
+    many = tmp_path / "many"
+    r1 = atlas.create(many_netcdf_files, str(one))
+    r4 = atlas.create(many_netcdf_files, str(many), workers=4)
+
+    assert r1["written"] == r4["written"]
+    a, b = atlas.info(str(one)), atlas.info(str(many))
+    assert a["array_stats"] == b["array_stats"]
+    assert a["dataset_count"] == b["dataset_count"]
+    assert a["interned_schemas"] == b["interned_schemas"]
+
+    # Every dataset still reads back with its own values.
+    for name in atlas.list_datasets(str(many)):
+        assert atlas.describe(str(many), name)["attributes"] == atlas.describe(
+            str(one), name
+        )["attributes"]
+
+
+def test_workers_still_stop_on_a_bad_file(many_netcdf_files, tmp_path):
+    xr.Dataset({"flag": xr.DataArray(np.array([True]), dims=["x"])}).to_netcdf(
+        many_netcdf_files / "bad.nc"
+    )
+    with pytest.raises(atlas.AtlasError):
+        atlas.create(many_netcdf_files, str(tmp_path / "c"), workers=4)
+
+
+def test_workers_can_skip_a_bad_file(many_netcdf_files, tmp_path):
+    xr.Dataset({"flag": xr.DataArray(np.array([True]), dims=["x"])}).to_netcdf(
+        many_netcdf_files / "bad.nc"
+    )
+    result = atlas.create(
+        many_netcdf_files, str(tmp_path / "c"), workers=4, on_error="skip"
+    )
+    assert [s["file"].endswith("bad.nc") for s in result["skipped"]] == [True]
+    assert len(result["written"]) == 12
+    # The order of what landed still follows the input.
+    assert atlas.list_datasets(str(tmp_path / "c")) == sorted(result["written"])
+
+
+def test_every_netcdf_read_stays_on_one_thread(many_netcdf_files, tmp_path, monkeypatch):
+    """netCDF4 sits on HDF5, which is not thread safe.
+
+    xarray locks an array read but not a variable open, so two threads inside
+    `open_dataset` segfault. Reads therefore stay on the calling thread, and
+    only the commit goes to the pool.
+    """
+    import threading
+
+    seen: set[int] = set()
+    original = xr.open_dataset
+
+    def spy(*args, **kwargs):
+        seen.add(threading.get_ident())
+        return original(*args, **kwargs)
+
+    monkeypatch.setattr(xr, "open_dataset", spy)
+    atlas.create(many_netcdf_files, str(tmp_path / "c"), workers=4)
+
+    assert len(seen) == 1, f"netCDF was read from {len(seen)} threads"
+    assert seen == {threading.get_ident()}, "a read left the calling thread"
+
+
+def test_workers_below_one_is_refused(netcdf_dir, tmp_path):
+    with pytest.raises(atlas.AtlasError, match="workers must be at least 1"):
+        atlas.create(netcdf_dir, str(tmp_path / "c"), workers=0)
+
+
 # ── unsupported dtypes ───────────────────────────────────────────────
 
 
@@ -474,6 +573,30 @@ def test_chunk_size_controls_the_stored_chunk_shape(tmp_path):
     assert large == [1024, 1024]
 
 
+def test_auto_opens_a_small_file_whole_and_a_large_one_in_blocks(tmp_path):
+    """`auto` picks a strategy per file. dask costs more than it saves on a
+    small one, and the stored layout comes out the same either way."""
+    from atlas._ops import _open_kwargs_for
+
+    budget = 128 * 1024 * 1024
+    base = {"chunks": "auto"}
+
+    small = tmp_path / "small.nc"
+    small.write_bytes(b"x" * 1024)
+    assert "chunks" not in _open_kwargs_for(small, "auto", base, budget)
+
+    large = tmp_path / "large.nc"
+    large.write_bytes(b"x" * (64 * 1024 * 1024))
+    assert _open_kwargs_for(large, "auto", base, budget) == base
+
+    # Every other mode is an explicit choice, and passes through untouched.
+    assert _open_kwargs_for(small, "native", {"chunks": {}}, budget) == {"chunks": {}}
+    assert _open_kwargs_for(small, None, {}, budget) == {}
+
+    # A smaller budget pulls the threshold down with it.
+    assert _open_kwargs_for(small, "auto", base, 1024) == base
+
+
 def test_small_files_still_land_as_a_single_chunk(netcdf_dir, tmp_path):
     """Auto chunking must not split an array that fits with room to spare."""
     atlas.create(netcdf_dir, str(tmp_path / "c"))

{atlas_python-0.16.3 → atlas_python-0.16.4}/docs/write-path.md

@@ -97,6 +97,12 @@ Concurrent datasets therefore land in finish order, and never interleave their
 bytes. `tests/integration.rs` asserts that the segments still tile the
 container without a gap under concurrent staging.
 
+**Ordinals do not follow that order.** Each dataset carries the number of the
+`add_dataset` call that opened it. `AtlasWriter::finish` then sorts the footer
+entries on that number. Stage a directory twice and every dataset lands at the same
+ordinal, however many threads did the work. Each entry holds its own byte
+range, so the segments need no matching order on disk.
+
 ## Failure
 
 | What happens | Result |

{atlas_python-0.16.3 → atlas_python-0.16.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "atlas-python"
-version = "0.16.3"
+version = "0.16.4"
 description = "Python bindings for the ATLAS array store"
 readme = "README.md"
 requires-python = ">=3.10"

{atlas_python-0.16.3 → atlas_python-0.16.4}/python/atlas/__init__.pyi

@@ -64,6 +64,7 @@ def create(
     chunk_size: str = "128MiB",
     decode_times: bool = True,
     convert_calendar: bool = False,
+    workers: int = 1,
     on_error: str = "stop",
     on_unsupported: str = "stop",
     progress: Optional[Callable[[str], None]] = None,
@@ -126,6 +127,12 @@ def create(
             Gregorian ``1973-03-10`` that names the same moment. A calendar
             with no real instant, such as ``360_day``, and a date outside the
             nanosecond range both raise instead.
+        workers: Commit this many datasets at once. The commit is the costly
+            half of an ingest. It is pure Rust and releases the GIL, so it
+            scales to about four times on a many-core machine. Every netCDF
+            read stays on the calling thread, because HDF5 is not thread safe.
+            Ordinals do not move, because ``add_dataset`` runs in file order
+            whatever the workers do.
         on_error: ``"stop"`` is the default. It abandons the whole collection
             on the first bad file. ``"skip"`` records that file and continues.
         on_unsupported: What one array of an unsupported dtype costs.

{atlas_python-0.16.3 → atlas_python-0.16.4}/python/atlas/_cli.py

@@ -145,24 +145,41 @@ def _parse_open_chunks(value: str) -> Any:
 def cmd_create(args: argparse.Namespace) -> int:
     chunks = json.loads(args.chunks) if args.chunks else None
     open_chunks = _parse_open_chunks(args.open_chunks)
+    recursive = not args.no_recursive
+
+    # A second walk of the tree, so each line can say how many remain. The
+    # walk only stats the files, so it costs little beside the ingest.
+    total = len(_ops.find_netcdf_files(args.directory, recursive=recursive))
+    width = len(str(total))
+    done = 0
 
     def progress(name: str) -> None:
+        nonlocal done
+        done += 1
         if not args.quiet:
-            print(f"  {name}", file=sys.stderr)
+            left = total - done
+            print(
+                f"  [{done:>{width}}/{total}] {name}  ({left} left)",
+                file=sys.stderr,
+            )
 
     if not args.quiet:
-        print(f"Writing {args.destination}", file=sys.stderr)
+        print(
+            f"Writing {args.destination} from {total} file(s)",
+            file=sys.stderr,
+        )
 
     result = _ops.create(
         args.directory,
         args.destination,
-        recursive=not args.no_recursive,
+        recursive=recursive,
         codec=args.codec,
         chunks=chunks,
         open_chunks=open_chunks,
         chunk_size=args.chunk_size,
         decode_times=not args.no_decode_times,
         convert_calendar=args.convert_calendar,
+        workers=args.workers,
         on_error="skip" if args.skip_errors else "stop",
         on_unsupported="skip" if args.skip_unsupported else "stop",
         progress=progress,
@@ -362,6 +379,17 @@ def build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="skip files that fail instead of abandoning the collection",
     )
+    p.add_argument(
+        "-j",
+        "--workers",
+        type=int,
+        default=1,
+        metavar="N",
+        help=(
+            "commit N datasets at once. It scales to about four times on a "
+            "many-core machine. Ordinals do not move"
+        ),
+    )
     p.add_argument(
         "--convert-calendar",
         action="store_true",
@@ -457,10 +485,12 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     args = build_parser().parse_args(argv)
     if args.log_file:
         try:
-            _log.log_to_file(args.log_file)
+            handler = _log.log_to_file(args.log_file)
         except OSError as exc:
             print(f"atlas: cannot open log file: {exc}", file=sys.stderr)
             return 1
+        # Say where it went, so nobody has to guess.
+        print(f"atlas: logging to {handler.baseFilename}", file=sys.stderr)
         _LOG.info("atlas %s: %s", __version__, " ".join(argv or sys.argv[1:]))
     try:
         return args.func(args)