atlas-python 0.16.2__tar.gz → 0.16.4__tar.gz

{atlas_python-0.16.2 → atlas_python-0.16.4}/Cargo.lock

@@ -71,7 +71,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-python"
-version = "0.16.2"
+version = "0.16.4"
 dependencies = [
  "atlas-rust",
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
 
 [[package]]
 name = "atlas-rust"
-version = "0.16.2"
+version = "0.16.4"
 dependencies = [
  "array-format",
  "async-trait",

{atlas_python-0.16.2 → atlas_python-0.16.4}/Cargo.toml

@@ -5,7 +5,7 @@ members = ["atlas-python"]
 # Registry name on crates.io (`atlas` is taken). The library is still
 # imported as `atlas` — see `[lib]` below.
 name = "atlas-rust"
-version = "0.16.2"
+version = "0.16.4"
 edition = "2024"
 description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
 license = "Apache-2.0"

{atlas_python-0.16.2 → atlas_python-0.16.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atlas-python
-Version: 0.16.2
+Version: 0.16.4
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "atlas-python"
-version = "0.16.2"
+version = "0.16.4"
 edition = "2021"
 readme = "README.md"
 

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/docs/cli.md

@@ -25,12 +25,20 @@ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
 
 ## Logging
 
-`--log-file PATH` appends every error and warning to a file, with the reason:
+`--log-file PATH` appends to the file you name. There is no default location,
+and no log at all without the flag. The command prints the absolute path it
+opened, so it is never a guess:
+
+```text
+atlas: logging to /home/you/ingest.log
+```
+
+The file gets every error and warning, with the reason:
 
 ```bash
 $ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
 $ cat ingest.log
-2026-09-01 14:30:41 INFO    atlas.cli: atlas 0.16.2: create /data/nc ...
+2026-09-01 14:30:41 INFO    atlas.cli: atlas 0.16.4: create /data/nc ...
 2026-09-01 14:30:41 INFO    atlas.ops: ingesting 2 file(s) into /data/collection
 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
 2026-09-01 14:30:41 INFO    atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
@@ -66,10 +74,25 @@ A failure part-way leaves no collection, and not a partial one.
 | `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
 | `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
 | `--skip-errors` | Skip files that fail instead of abandoning the collection |
+| `-j`, `--workers N` | Stage N files at once. About 3x on a many-core machine. Ordinals do not move |
+| `--convert-calendar` | Turn a cftime axis into exact Gregorian timestamps, each keeping its instant |
+| `--no-decode-times` | Keep a time axis as raw numbers, for a calendar that decodes to cftime |
 | `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
 | `-q`, `--quiet` | Do not list a file as it lands |
 
-Progress goes to stderr, so a pipe still reads stdout.
+Progress goes to stderr, so a pipe still reads stdout. Each line counts the
+files and says how many remain:
+
+```text
+Writing /data/collection from 3 file(s)
+  [1/3] 2024-01.nc  (2 left)
+  [2/3] 2024-02.nc  (1 left)
+  [3/3] 2024-03.nc  (0 left)
+3 dataset(s) written to /data/collection
+```
+
+`-q` turns the per-file lines off. The same counter goes to `--log-file`, for
+a run nobody watched.
 
 ### Unsupported dtypes
 
@@ -109,7 +132,7 @@ size. `--open-chunks` picks another strategy:
 
 | Mode | Reads | Stored chunk shape |
 |---|---|---|
-| `auto` *(default)* | blocks sized to `--chunk-size` | those blocks |
+| `auto` *(default)* | whole when the file is small, else blocks sized to `--chunk-size` | those blocks |
 | `native` | the file's own chunk encoding | that encoding |
 | `none` | each variable whole | one full-shape chunk |
 | JSON dict | as given, per dimension | as given |

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/docs/guides/creating.md

@@ -140,8 +140,10 @@ that stream to stderr. See [Installation](../installation.md).
 These are one decision. The blocks a file *reads* in are the chunks it
 *stores* in.
 
-Each file opens with dask chunking. `open_chunks="auto"` is the default. dask
-then sizes the blocks to `chunk_size`, which is 128 MiB. Three results follow:
+`open_chunks="auto"` is the default, and it picks a strategy per file. A file
+well inside the block budget opens whole. The dask graph costs about 40 ms
+per file, and a small variable lands as one chunk either way. A larger
+file streams through dask, so memory stays bounded. Three results follow:
 
 - A file far larger than memory streams block by block, and does not read
   whole.
@@ -158,13 +160,14 @@ atlas.create("/data/nc", dest, chunk_size="64MiB")
 ```
 
 `chunk_size` is about the memory ceiling per variable. Lower it on a small
-machine. Raise it for larger stored chunks.
+machine. Raise it for larger stored chunks. It also moves the threshold at
+which `"auto"` stops opening a file whole.
 
 ### How files are opened
 
 | `open_chunks` | Reads | Stored chunk shape |
 |---|---|---|
-| `"auto"` *(default)* | blocks sized to `chunk_size` | those blocks |
+| `"auto"` *(default)* | whole when small, else blocks sized to `chunk_size` | those blocks |
 | `"native"` | the file's own chunk encoding | that encoding |
 | `None` | each variable whole | one full-shape chunk |
 | `{"time": 100}` | as given, per dimension | as given |
@@ -206,6 +209,51 @@ arrays["temperature"]["chunk_shape"]
 atlas show dest 2024-01.nc | grep _ChunkShape
 ```
 
+### Speed on many small files
+
+A directory of small files spends most of its time per file, not per byte. On
+213 KiB profile files the default runs at about 30 files per second, or five
+minutes for ten thousand.
+
+**`--workers N` stages N files at once.** It is the largest single win:
+
+```bash
+atlas create /data/nc /data/collection --workers 4
+```
+
+```text
+workers=1 :  32 files/s   1.00x
+workers=2 :  84 files/s   2.65x
+workers=4 : 120 files/s   3.77x
+workers=8 : 118 files/s   3.72x   <- plateau
+```
+
+The commit is the costly half of an ingest, about two thirds of it. It is pure
+Rust, holds no lock until its append, and releases the GIL, so it overlaps.
+The reads stay on the calling thread, and they are what flattens the curve
+near four.
+
+The reads have to stay there. netCDF4 sits on HDF5, which is not thread safe,
+and xarray locks an array read but not a variable open. Two threads inside
+`open_dataset` crash the process.
+
+Nothing else changes. `add_dataset` runs in file order, so every ordinal
+matches a sequential build. The summary sorts back into file order too. Only
+`progress` reports in completion order.
+
+Two more settings matter when that is still too slow:
+
+- **`--open-chunks native`** forces dask on every file. That costs about twice
+  as long on a small file. Use it only when the file's own chunking is what
+  you want stored.
+- **`--codec none`** trades size for speed. It saved about 20 percent on the
+  same files.
+
+Past the plateau, run several `atlas create` commands at once, one per part of
+the tree. Each has its own writer, so nothing serialises between them. That
+reached 3.7x on eight processes where threads reached 2.9x. The cost is one
+collection per part.
+
 ### The writer's own memory
 
 Staging runs on local disk. `array-format` spills each compressed chunk to a
@@ -233,8 +281,15 @@ used.
 atlas.create("/data/nc", dest, progress=lambda name: print(name))
 ```
 
-The CLI does this by default, to stderr, so a pipe still reads stdout. `-q`
-turns it off.
+The CLI does this by default, to stderr, so a pipe still reads stdout. Each
+line counts the files and says how many remain:
+
+```text
+  [ 12/10000] 000043_CFPOINT_3593_V0.nc  (9988 left)
+```
+
+`-q` turns the per-file lines off. Pass `--log-file PATH` to keep the same
+counter in a file. The command prints the absolute path it opened.
 
 ## What can go wrong
 

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/docs/guides/dtypes.md

@@ -17,6 +17,71 @@ What a NetCDF variable becomes when `atlas create` ingests it.
 `atlas show` prints the atlas name, so a `datetime64[ns]` variable appears as
 `timestamp_nanoseconds`.
 
+### Calendars that decode to cftime
+
+xarray decodes a time axis to `datetime64[ns]` only when the calendar allows
+it. A Julian, `360_day`, or `noleap` calendar, or a date outside the
+`datetime64[ns]` range, decodes to a `cftime` object instead. Those arrive as
+a numpy `object` array, and atlas cannot store one:
+
+```text
+atlas: profile.nc: variable 'JULD' holds cftime objects (DatetimeJulian),
+which atlas cannot store...
+```
+
+Atlas refuses by default rather than guess, because two conversions exist and
+they differ by 13 days. Pick one.
+
+**Convert to the exact instant.** This is almost always what you want:
+
+```bash
+atlas create /data/nc /data/collection --convert-calendar
+```
+
+```python
+atlas.create("/data/nc", dest, convert_calendar=True)
+```
+
+The axis becomes `timestamp_nanoseconds`, and every value keeps the moment it
+named. A Julian `1973-02-25 18:15` becomes the Gregorian `1973-03-10 18:15`.
+Those are one instant under two calendars. cftime maps between them through
+the Julian Day, so nothing is approximated.
+
+**Keep the raw numbers.** The axis stores as an integer, with its `units` and
+`calendar` attributes beside it. A reader decodes it later with
+`cftime.num2date`:
+
+```bash
+atlas create /data/nc /data/collection --no-decode-times
+```
+
+**Drop the axis.** `--skip-unsupported` leaves the time array out, and keeps
+the rest of the dataset.
+
+#### Two conversions, 13 days apart
+
+The Julian calendar takes a leap year every four years. The Gregorian calendar
+drops three of those every four centuries. The gap grows by about three days
+per 400 years, and holds at 13 days from 1900 to 2100.
+
+| Conversion | Julian 2024-01-01 becomes | It keeps |
+|---|---|---|
+| `--convert-calendar` | 2024-01-14 | the instant |
+| `xr.Dataset.convert_calendar("standard")` | 2024-01-01 | the labels |
+
+The xarray method keeps the year, month, and day, which moves the moment by 13
+days. Use it only when the labels matter more than the instant.
+
+#### Two limits
+
+**An artificial calendar names no instant.** A `360_day`, `noleap`, or
+`all_leap` year is a model convention with no place on a real timeline.
+`--convert-calendar` raises for one. Use `--no-decode-times` instead.
+
+**A nanosecond timestamp spans 1677-09-21 to 2262-04-11.** A date outside that
+raises. numpy wraps such a date in silence, so atlas checks every value before
+it stores one.
+
 ### datetime and timedelta
 
 Atlas supports the `[ns]` resolution of `datetime64` alone. It rejects every

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/src/writer.rs

@@ -322,7 +322,7 @@ impl PyDatasetWriter {
                     String::from_utf8_lossy(&b).into_owned()
                 } else {
                     return Err(PyTypeError::new_err(format!(
-                        "string array element must be str or bytes, got {:?}",
+                        "array {name:?}: a string element must be str or bytes, got {:?}",
                         bound.get_type().name()?
                     )));
                 };

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/tests/test_cli.py

@@ -186,6 +186,37 @@ def test_create_descends_by_default_and_no_recursive_opts_out(capsys, tmp_path):
     assert code == 0, err
 
 
+def test_create_counts_progress_and_what_is_left(capsys, netcdf_dir, tmp_path):
+    code, _, err = run(capsys, "create", str(netcdf_dir), str(tmp_path / "c"))
+    assert code == 0
+    assert "from 3 file(s)" in err
+    assert "[1/3] 2024-01.nc  (2 left)" in err
+    assert "[3/3] 2024-03.nc  (0 left)" in err
+
+
+def test_log_file_says_where_it_writes(capsys, netcdf_dir, tmp_path):
+    log = tmp_path / "run.log"
+    code, _, err = run(
+        capsys, "create", str(netcdf_dir), str(tmp_path / "c"), "-q",
+        "--log-file", str(log),
+    )
+    assert code == 0
+    assert f"logging to {log}" in err
+    # The log carries the same counter, for a run nobody watched.
+    text = log.read_text()
+    assert "[1/3] wrote 2024-01.nc" in text
+    assert "[3/3] wrote 2024-03.nc" in text
+
+
+def test_workers_flag_builds_the_same_collection(capsys, netcdf_dir, tmp_path):
+    run(capsys, "create", str(netcdf_dir), str(tmp_path / "one"), "-q")
+    run(capsys, "create", str(netcdf_dir), str(tmp_path / "many"), "-q", "-j", "4")
+
+    code, a, _ = run(capsys, "ls", str(tmp_path / "one"))
+    code, b, _ = run(capsys, "ls", str(tmp_path / "many"))
+    assert a.split() == b.split() == ["2024-01.nc", "2024-02.nc", "2024-03.nc"]
+
+
 # ── ls ───────────────────────────────────────────────────────────────
 
 

{atlas_python-0.16.2 → atlas_python-0.16.4}/atlas-python/tests/test_ops.py

@@ -148,6 +148,105 @@ def test_every_codec_round_trips(netcdf_dir, tmp_path, codec):
     assert len(atlas.list_datasets(str(dest))) == 3
 
 
+# ── parallel staging ─────────────────────────────────────────────────
+
+
+@pytest.fixture
+def many_netcdf_files(tmp_path):
+    """Enough files that several workers actually overlap."""
+    d = tmp_path / "many"
+    d.mkdir()
+    for i in range(12):
+        make_dataset((i % 12) + 1).to_netcdf(d / f"m{i:03d}.nc")
+    return d
+
+
+@pytest.mark.parametrize("workers", [2, 4])
+def test_workers_keep_the_ordinals_of_a_sequential_build(
+    many_netcdf_files, tmp_path, workers
+):
+    """add_dataset runs in file order, so a worker cannot move an ordinal."""
+    one = tmp_path / "one"
+    many = tmp_path / f"many{workers}"
+    atlas.create(many_netcdf_files, str(one))
+    atlas.create(many_netcdf_files, str(many), workers=workers)
+
+    assert atlas.list_datasets(str(one)) == atlas.list_datasets(str(many))
+    for name in atlas.list_datasets(str(one)):
+        assert (
+            atlas.describe(str(one), name)["ordinal"]
+            == atlas.describe(str(many), name)["ordinal"]
+        )
+
+
+def test_workers_produce_the_same_collection(many_netcdf_files, tmp_path):
+    one = tmp_path / "one"
+    many = tmp_path / "many"
+    r1 = atlas.create(many_netcdf_files, str(one))
+    r4 = atlas.create(many_netcdf_files, str(many), workers=4)
+
+    assert r1["written"] == r4["written"]
+    a, b = atlas.info(str(one)), atlas.info(str(many))
+    assert a["array_stats"] == b["array_stats"]
+    assert a["dataset_count"] == b["dataset_count"]
+    assert a["interned_schemas"] == b["interned_schemas"]
+
+    # Every dataset still reads back with its own values.
+    for name in atlas.list_datasets(str(many)):
+        assert atlas.describe(str(many), name)["attributes"] == atlas.describe(
+            str(one), name
+        )["attributes"]
+
+
+def test_workers_still_stop_on_a_bad_file(many_netcdf_files, tmp_path):
+    xr.Dataset({"flag": xr.DataArray(np.array([True]), dims=["x"])}).to_netcdf(
+        many_netcdf_files / "bad.nc"
+    )
+    with pytest.raises(atlas.AtlasError):
+        atlas.create(many_netcdf_files, str(tmp_path / "c"), workers=4)
+
+
+def test_workers_can_skip_a_bad_file(many_netcdf_files, tmp_path):
+    xr.Dataset({"flag": xr.DataArray(np.array([True]), dims=["x"])}).to_netcdf(
+        many_netcdf_files / "bad.nc"
+    )
+    result = atlas.create(
+        many_netcdf_files, str(tmp_path / "c"), workers=4, on_error="skip"
+    )
+    assert [s["file"].endswith("bad.nc") for s in result["skipped"]] == [True]
+    assert len(result["written"]) == 12
+    # The order of what landed still follows the input.
+    assert atlas.list_datasets(str(tmp_path / "c")) == sorted(result["written"])
+
+
+def test_every_netcdf_read_stays_on_one_thread(many_netcdf_files, tmp_path, monkeypatch):
+    """netCDF4 sits on HDF5, which is not thread safe.
+
+    xarray locks an array read but not a variable open, so two threads inside
+    `open_dataset` segfault. Reads therefore stay on the calling thread, and
+    only the commit goes to the pool.
+    """
+    import threading
+
+    seen: set[int] = set()
+    original = xr.open_dataset
+
+    def spy(*args, **kwargs):
+        seen.add(threading.get_ident())
+        return original(*args, **kwargs)
+
+    monkeypatch.setattr(xr, "open_dataset", spy)
+    atlas.create(many_netcdf_files, str(tmp_path / "c"), workers=4)
+
+    assert len(seen) == 1, f"netCDF was read from {len(seen)} threads"
+    assert seen == {threading.get_ident()}, "a read left the calling thread"
+
+
+def test_workers_below_one_is_refused(netcdf_dir, tmp_path):
+    with pytest.raises(atlas.AtlasError, match="workers must be at least 1"):
+        atlas.create(netcdf_dir, str(tmp_path / "c"), workers=0)
+
+
 # ── unsupported dtypes ───────────────────────────────────────────────
 
 
@@ -215,6 +314,151 @@ def test_a_clean_ingest_reports_no_skipped_arrays(netcdf_dir, tmp_path):
     assert result["skipped_arrays"] == []
 
 
+# ── cftime calendars ─────────────────────────────────────────────────
+
+
+@pytest.fixture
+def netcdf_dir_with_a_julian_calendar(tmp_path):
+    """A time axis xarray decodes to cftime, not to datetime64."""
+    import cftime
+
+    d = tmp_path / "nc"
+    d.mkdir()
+    times = [cftime.DatetimeJulian(2024, 1, i + 1) for i in range(4)]
+    xr.Dataset(
+        {"temp": ("time", np.arange(4, dtype=np.float32))},
+        coords={"time": ("time", times)},
+    ).to_netcdf(d / "julian.nc")
+    return d
+
+
+def test_a_cftime_axis_reports_the_variable_and_the_remedy(
+    netcdf_dir_with_a_julian_calendar, tmp_path
+):
+    with pytest.raises(atlas.AtlasError) as exc:
+        atlas.create(netcdf_dir_with_a_julian_calendar, str(tmp_path / "c"))
+
+    message = str(exc.value)
+    assert "'time'" in message, "the message must name the variable"
+    assert "DatetimeJulian" in message
+    assert "decode_times=False" in message
+
+
+def test_decode_times_false_stores_a_cftime_axis_as_raw_numbers(
+    netcdf_dir_with_a_julian_calendar, tmp_path
+):
+    dest = tmp_path / "c"
+    atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), decode_times=False)
+
+    arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
+    assert arrays["time"]["dtype"] == "int64"
+    # The units and the calendar survive, so a reader can decode the numbers.
+    assert arrays["time"]["attributes"]["calendar"] == "julian"
+    assert "days since" in arrays["time"]["attributes"]["units"]
+    assert arrays["temp"]["dtype"] == "float32"
+
+
+def test_a_cftime_axis_can_be_skipped_per_array(
+    netcdf_dir_with_a_julian_calendar, tmp_path
+):
+    dest = tmp_path / "c"
+    result = atlas.create(
+        netcdf_dir_with_a_julian_calendar, str(dest), on_unsupported="skip"
+    )
+
+    assert [s["array"] for s in result["skipped_arrays"]] == ["time"]
+    assert [a["name"] for a in atlas.describe(str(dest), "julian.nc")["arrays"]] == [
+        "temp"
+    ]
+
+
+def test_convert_calendar_keeps_the_exact_instant(
+    netcdf_dir_with_a_julian_calendar, tmp_path
+):
+    """A Julian label and its Gregorian label name one moment, 13 days apart."""
+    dest = tmp_path / "c"
+    atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), convert_calendar=True)
+
+    arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
+    assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
+
+    # The fixture holds Julian 2024-01-01 through 2024-01-04. The same instants
+    # carry the Gregorian labels 2024-01-14 through 2024-01-17.
+    first = np.datetime64(arrays["time"]["stats"]["min"], "ns")
+    last = np.datetime64(arrays["time"]["stats"]["max"], "ns")
+    assert first == np.datetime64("2024-01-14T00:00:00")
+    assert last == np.datetime64("2024-01-17T00:00:00")
+
+
+def test_convert_calendar_matches_a_per_element_conversion(
+    netcdf_dir_with_a_julian_calendar,
+):
+    """The array-at-once conversion must equal the obvious slow one."""
+    from atlas.xarray import _cftime_to_datetime64
+
+    ds = xr.open_dataset(netcdf_dir_with_a_julian_calendar / "julian.nc")
+    values = np.asarray(ds["time"].values)
+
+    fast = _cftime_to_datetime64("time", values)
+    slow = np.array(
+        [
+            np.datetime64(v.change_calendar("proleptic_gregorian").isoformat(), "ns")
+            for v in values
+        ]
+    )
+    assert np.array_equal(fast, slow)
+
+
+def test_an_artificial_calendar_has_no_exact_gregorian_date(tmp_path):
+    """A 360_day year names no real instant, so no conversion exists."""
+    import cftime
+
+    from atlas.xarray import _cftime_to_datetime64
+
+    values = np.array([cftime.Datetime360Day(2024, 2, 30)], dtype=object)
+    with pytest.raises(NotImplementedError, match="360_day"):
+        _cftime_to_datetime64("time", values)
+
+
+def test_a_date_outside_the_nanosecond_range_is_refused_not_wrapped(tmp_path):
+    """numpy wraps such a date in silence. Atlas must not pass one on."""
+    import cftime
+
+    from atlas.xarray import _cftime_to_datetime64
+
+    for year in (1600, 2300):
+        values = np.array([cftime.DatetimeJulian(year, 1, 1)], dtype=object)
+        with pytest.raises(NotImplementedError, match="outside the range"):
+            _cftime_to_datetime64("time", values)
+
+
+def test_convert_calendar_leaves_a_normal_time_axis_alone(netcdf_dir, tmp_path):
+    dest = tmp_path / "c"
+    atlas.create(netcdf_dir, str(dest), convert_calendar=True)
+    arrays = {a["name"]: a for a in atlas.describe(str(dest), "2024-01.nc")["arrays"]}
+    assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
+
+
+def test_an_object_array_of_something_else_is_refused_clearly(tmp_path):
+    """numpy reports `object` for both a string array and this one."""
+    from atlas.xarray import _reject_unstorable_object_array
+
+    var = xr.DataArray(np.array([{"a": 1}, {"b": 2}], dtype=object), dims=["x"])
+    with pytest.raises(NotImplementedError, match="object array of dict"):
+        _reject_unstorable_object_array("payload", var)
+
+
+def test_a_real_string_array_still_passes_the_object_check(tmp_path):
+    from atlas.xarray import _reject_unstorable_object_array
+
+    # A missing cell must not be mistaken for an unsupported element.
+    var = xr.DataArray(np.array([None, "b", "c"], dtype=object), dims=["x"])
+    _reject_unstorable_object_array("label", var)
+
+    empty = xr.DataArray(np.array([], dtype=object), dims=["x"])
+    _reject_unstorable_object_array("empty", empty)
+
+
 # ── the log file ─────────────────────────────────────────────────────
 
 
@@ -329,6 +573,30 @@ def test_chunk_size_controls_the_stored_chunk_shape(tmp_path):
     assert large == [1024, 1024]
 
 
+def test_auto_opens_a_small_file_whole_and_a_large_one_in_blocks(tmp_path):
+    """`auto` picks a strategy per file. dask costs more than it saves on a
+    small one, and the stored layout comes out the same either way."""
+    from atlas._ops import _open_kwargs_for
+
+    budget = 128 * 1024 * 1024
+    base = {"chunks": "auto"}
+
+    small = tmp_path / "small.nc"
+    small.write_bytes(b"x" * 1024)
+    assert "chunks" not in _open_kwargs_for(small, "auto", base, budget)
+
+    large = tmp_path / "large.nc"
+    large.write_bytes(b"x" * (64 * 1024 * 1024))
+    assert _open_kwargs_for(large, "auto", base, budget) == base
+
+    # Every other mode is an explicit choice, and passes through untouched.
+    assert _open_kwargs_for(small, "native", {"chunks": {}}, budget) == {"chunks": {}}
+    assert _open_kwargs_for(small, None, {}, budget) == {}
+
+    # A smaller budget pulls the threshold down with it.
+    assert _open_kwargs_for(small, "auto", base, 1024) == base
+
+
 def test_small_files_still_land_as_a_single_chunk(netcdf_dir, tmp_path):
     """Auto chunking must not split an array that fits with room to spare."""
     atlas.create(netcdf_dir, str(tmp_path / "c"))

{atlas_python-0.16.2 → atlas_python-0.16.4}/docs/write-path.md

@@ -97,6 +97,12 @@ Concurrent datasets therefore land in finish order, and never interleave their
 bytes. `tests/integration.rs` asserts that the segments still tile the
 container without a gap under concurrent staging.
 
+**Ordinals do not follow that order.** Each dataset carries the number of the
+`add_dataset` call that opened it. `AtlasWriter::finish` then sorts the footer
+entries on that number. Stage a directory twice and every dataset lands at the same
+ordinal, however many threads did the work. Each entry holds its own byte
+range, so the segments need no matching order on disk.
+
 ## Failure
 
 | What happens | Result |

{atlas_python-0.16.2 → atlas_python-0.16.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "atlas-python"
-version = "0.16.2"
+version = "0.16.4"
 description = "Python bindings for the ATLAS array store"
 readme = "README.md"
 requires-python = ">=3.10"

{atlas_python-0.16.2 → atlas_python-0.16.4}/python/atlas/__init__.pyi

@@ -62,6 +62,9 @@ def create(
     chunks: Optional[dict[str, Sequence[int]]] = None,
     open_chunks: Union[str, dict[str, int], None] = "auto",
     chunk_size: str = "128MiB",
+    decode_times: bool = True,
+    convert_calendar: bool = False,
+    workers: int = 1,
     on_error: str = "stop",
     on_unsupported: str = "stop",
     progress: Optional[Callable[[str], None]] = None,
@@ -114,6 +117,22 @@ def create(
         chunk_size: The block size ``"auto"`` aims at, as a dask size string.
             It is about the memory ceiling per variable during ingest. It
             defaults to ``"128MiB"``.
+        decode_times: How xarray reads a time axis. Under the default, a
+            calendar it cannot map to ``datetime64[ns]``, such as a Julian
+            one, decodes to cftime objects, which atlas cannot store. Set it
+            false to keep the raw numbers, with their ``units`` and
+            ``calendar`` attributes.
+        convert_calendar: Turn a cftime axis into exact Gregorian timestamps.
+            Each one keeps its instant, so a Julian ``1973-02-25`` becomes the
+            Gregorian ``1973-03-10`` that names the same moment. A calendar
+            with no real instant, such as ``360_day``, and a date outside the
+            nanosecond range both raise instead.
+        workers: Commit this many datasets at once. The commit is the costly
+            half of an ingest. It is pure Rust and releases the GIL, so it
+            scales to about four times on a many-core machine. Every netCDF
+            read stays on the calling thread, because HDF5 is not thread safe.
+            Ordinals do not move, because ``add_dataset`` runs in file order
+            whatever the workers do.
         on_error: ``"stop"`` is the default. It abandons the whole collection
             on the first bad file. ``"skip"`` records that file and continues.
         on_unsupported: What one array of an unsupported dtype costs.