atlas-python 0.16.2__tar.gz → 0.16.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atlas_python-0.16.2 → atlas_python-0.16.3}/Cargo.lock +2 -2
- {atlas_python-0.16.2 → atlas_python-0.16.3}/Cargo.toml +1 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/PKG-INFO +1 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/Cargo.toml +1 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/cli.md +2 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/dtypes.md +65 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/writer.rs +1 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_ops.py +145 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/pyproject.toml +1 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__init__.pyi +12 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__main__.py +3 -3
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_cli.py +19 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_ops.py +22 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/xarray.py +160 -1
- {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/fixtures/from_python/data.atlas +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-python-docs.yaml +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-python-release.yaml +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-rust-release.yaml +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/ci.yaml +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/.gitignore +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/CONTRIBUTING.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/LICENSE +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/README.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/.python-version +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/LICENSE +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/README.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/examples.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/cloud-storage.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/creating.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/inspecting.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/reading-data.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/removing.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/index.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/installation.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/quickstart.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/reference/api.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/vs-zarr-netcdf.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/examples/01_library.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/examples/02_object_store.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/mkdocs.yml +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/attr.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/dtype.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/error.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/lib.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/logging.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/reader.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/runtime.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/source.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/conftest.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/make_fixture.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_cli.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_source.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/README.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/architecture.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/data-model.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/format.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/python.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/read-path.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/write-path.md +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/lifecycle.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/sensor_fleet.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/weather_store.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__init__.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_log.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_source.py +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/py.typed +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/config.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/error.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/footer.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/mask.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/mod.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/segment_store.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/lib.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/reader/mod.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/array.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/attr.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/dtype.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/mod.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/src/writer/mod.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/cross_fixture.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/fixtures/golden_v1/data.atlas +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/golden.rs +0 -0
- {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/integration.rs +0 -0
|
@@ -71,7 +71,7 @@ dependencies = [
|
|
|
71
71
|
|
|
72
72
|
[[package]]
|
|
73
73
|
name = "atlas-python"
|
|
74
|
-
version = "0.16.
|
|
74
|
+
version = "0.16.3"
|
|
75
75
|
dependencies = [
|
|
76
76
|
"atlas-rust",
|
|
77
77
|
"ndarray",
|
|
@@ -86,7 +86,7 @@ dependencies = [
|
|
|
86
86
|
|
|
87
87
|
[[package]]
|
|
88
88
|
name = "atlas-rust"
|
|
89
|
-
version = "0.16.
|
|
89
|
+
version = "0.16.3"
|
|
90
90
|
dependencies = [
|
|
91
91
|
"array-format",
|
|
92
92
|
"async-trait",
|
|
@@ -5,7 +5,7 @@ members = ["atlas-python"]
|
|
|
5
5
|
# Registry name on crates.io (`atlas` is taken). The library is still
|
|
6
6
|
# imported as `atlas` — see `[lib]` below.
|
|
7
7
|
name = "atlas-rust"
|
|
8
|
-
version = "0.16.
|
|
8
|
+
version = "0.16.3"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
|
|
11
11
|
license = "Apache-2.0"
|
|
@@ -30,7 +30,7 @@ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
|
|
|
30
30
|
```bash
|
|
31
31
|
$ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
|
|
32
32
|
$ cat ingest.log
|
|
33
|
-
2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.
|
|
33
|
+
2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.3: create /data/nc ...
|
|
34
34
|
2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
|
|
35
35
|
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
|
|
36
36
|
2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
|
|
@@ -66,6 +66,7 @@ A failure part-way leaves no collection, and not a partial one.
|
|
|
66
66
|
| `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
|
|
67
67
|
| `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
|
|
68
68
|
| `--skip-errors` | Skip files that fail instead of abandoning the collection |
|
|
69
|
+
| `--no-decode-times` | Keep a time axis as raw numbers, for a calendar that decodes to cftime |
|
|
69
70
|
| `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
|
|
70
71
|
| `-q`, `--quiet` | Do not list a file as it lands |
|
|
71
72
|
|
|
@@ -17,6 +17,71 @@ What a NetCDF variable becomes when `atlas create` ingests it.
|
|
|
17
17
|
`atlas show` prints the atlas name, so a `datetime64[ns]` variable appears as
|
|
18
18
|
`timestamp_nanoseconds`.
|
|
19
19
|
|
|
20
|
+
### Calendars that decode to cftime
|
|
21
|
+
|
|
22
|
+
xarray decodes a time axis to `datetime64[ns]` only when the calendar allows
|
|
23
|
+
it. A Julian, `360_day`, or `noleap` calendar, or a date outside the
|
|
24
|
+
`datetime64[ns]` range, decodes to a `cftime` object instead. Those arrive as
|
|
25
|
+
a numpy `object` array, and atlas cannot store one:
|
|
26
|
+
|
|
27
|
+
```text
|
|
28
|
+
atlas: profile.nc: variable 'JULD' holds cftime objects (DatetimeJulian),
|
|
29
|
+
which atlas cannot store...
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Atlas refuses by default rather than guess, because two conversions exist and
|
|
33
|
+
they differ by 13 days. Pick one.
|
|
34
|
+
|
|
35
|
+
**Convert to the exact instant.** This is almost always what you want:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
atlas create /data/nc /data/collection --convert-calendar
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
atlas.create("/data/nc", dest, convert_calendar=True)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The axis becomes `timestamp_nanoseconds`, and every value keeps the moment it
|
|
46
|
+
named. A Julian `1973-02-25 18:15` becomes the Gregorian `1973-03-10 18:15`.
|
|
47
|
+
Those are one instant under two calendars. cftime maps between them through
|
|
48
|
+
the Julian Day, so nothing is approximated.
|
|
49
|
+
|
|
50
|
+
**Keep the raw numbers.** The axis stores as an integer, with its `units` and
|
|
51
|
+
`calendar` attributes beside it. A reader decodes it later with
|
|
52
|
+
`cftime.num2date`:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
atlas create /data/nc /data/collection --no-decode-times
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
**Drop the axis.** `--skip-unsupported` leaves the time array out, and keeps
|
|
59
|
+
the rest of the dataset.
|
|
60
|
+
|
|
61
|
+
#### Two conversions, 13 days apart
|
|
62
|
+
|
|
63
|
+
The Julian calendar takes a leap year every four years. The Gregorian calendar
|
|
64
|
+
drops three of those every four centuries. The gap grows by about three days
|
|
65
|
+
per 400 years, and holds at 13 days from 1900 to 2100.
|
|
66
|
+
|
|
67
|
+
| Conversion | Julian 2024-01-01 becomes | It keeps |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| `--convert-calendar` | 2024-01-14 | the instant |
|
|
70
|
+
| `xr.Dataset.convert_calendar("standard")` | 2024-01-01 | the labels |
|
|
71
|
+
|
|
72
|
+
The xarray method keeps the year, month, and day, which moves the moment by 13
|
|
73
|
+
days. Use it only when the labels matter more than the instant.
|
|
74
|
+
|
|
75
|
+
#### Two limits
|
|
76
|
+
|
|
77
|
+
**An artificial calendar names no instant.** A `360_day`, `noleap`, or
|
|
78
|
+
`all_leap` year is a model convention with no place on a real timeline.
|
|
79
|
+
`--convert-calendar` raises for one. Use `--no-decode-times` instead.
|
|
80
|
+
|
|
81
|
+
**A nanosecond timestamp spans 1677-09-21 to 2262-04-11.** A date outside that
|
|
82
|
+
raises. numpy wraps such a date in silence, so atlas checks every value before
|
|
83
|
+
it stores one.
|
|
84
|
+
|
|
20
85
|
### datetime and timedelta
|
|
21
86
|
|
|
22
87
|
Atlas supports the `[ns]` resolution of `datetime64` alone. It rejects every
|
|
@@ -322,7 +322,7 @@ impl PyDatasetWriter {
|
|
|
322
322
|
String::from_utf8_lossy(&b).into_owned()
|
|
323
323
|
} else {
|
|
324
324
|
return Err(PyTypeError::new_err(format!(
|
|
325
|
-
"string
|
|
325
|
+
"array {name:?}: a string element must be str or bytes, got {:?}",
|
|
326
326
|
bound.get_type().name()?
|
|
327
327
|
)));
|
|
328
328
|
};
|
|
@@ -215,6 +215,151 @@ def test_a_clean_ingest_reports_no_skipped_arrays(netcdf_dir, tmp_path):
|
|
|
215
215
|
assert result["skipped_arrays"] == []
|
|
216
216
|
|
|
217
217
|
|
|
218
|
+
# ── cftime calendars ─────────────────────────────────────────────────
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
@pytest.fixture
|
|
222
|
+
def netcdf_dir_with_a_julian_calendar(tmp_path):
|
|
223
|
+
"""A time axis xarray decodes to cftime, not to datetime64."""
|
|
224
|
+
import cftime
|
|
225
|
+
|
|
226
|
+
d = tmp_path / "nc"
|
|
227
|
+
d.mkdir()
|
|
228
|
+
times = [cftime.DatetimeJulian(2024, 1, i + 1) for i in range(4)]
|
|
229
|
+
xr.Dataset(
|
|
230
|
+
{"temp": ("time", np.arange(4, dtype=np.float32))},
|
|
231
|
+
coords={"time": ("time", times)},
|
|
232
|
+
).to_netcdf(d / "julian.nc")
|
|
233
|
+
return d
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def test_a_cftime_axis_reports_the_variable_and_the_remedy(
|
|
237
|
+
netcdf_dir_with_a_julian_calendar, tmp_path
|
|
238
|
+
):
|
|
239
|
+
with pytest.raises(atlas.AtlasError) as exc:
|
|
240
|
+
atlas.create(netcdf_dir_with_a_julian_calendar, str(tmp_path / "c"))
|
|
241
|
+
|
|
242
|
+
message = str(exc.value)
|
|
243
|
+
assert "'time'" in message, "the message must name the variable"
|
|
244
|
+
assert "DatetimeJulian" in message
|
|
245
|
+
assert "decode_times=False" in message
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def test_decode_times_false_stores_a_cftime_axis_as_raw_numbers(
|
|
249
|
+
netcdf_dir_with_a_julian_calendar, tmp_path
|
|
250
|
+
):
|
|
251
|
+
dest = tmp_path / "c"
|
|
252
|
+
atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), decode_times=False)
|
|
253
|
+
|
|
254
|
+
arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
|
|
255
|
+
assert arrays["time"]["dtype"] == "int64"
|
|
256
|
+
# The units and the calendar survive, so a reader can decode the numbers.
|
|
257
|
+
assert arrays["time"]["attributes"]["calendar"] == "julian"
|
|
258
|
+
assert "days since" in arrays["time"]["attributes"]["units"]
|
|
259
|
+
assert arrays["temp"]["dtype"] == "float32"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def test_a_cftime_axis_can_be_skipped_per_array(
|
|
263
|
+
netcdf_dir_with_a_julian_calendar, tmp_path
|
|
264
|
+
):
|
|
265
|
+
dest = tmp_path / "c"
|
|
266
|
+
result = atlas.create(
|
|
267
|
+
netcdf_dir_with_a_julian_calendar, str(dest), on_unsupported="skip"
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
assert [s["array"] for s in result["skipped_arrays"]] == ["time"]
|
|
271
|
+
assert [a["name"] for a in atlas.describe(str(dest), "julian.nc")["arrays"]] == [
|
|
272
|
+
"temp"
|
|
273
|
+
]
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def test_convert_calendar_keeps_the_exact_instant(
|
|
277
|
+
netcdf_dir_with_a_julian_calendar, tmp_path
|
|
278
|
+
):
|
|
279
|
+
"""A Julian label and its Gregorian label name one moment, 13 days apart."""
|
|
280
|
+
dest = tmp_path / "c"
|
|
281
|
+
atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), convert_calendar=True)
|
|
282
|
+
|
|
283
|
+
arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
|
|
284
|
+
assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
|
|
285
|
+
|
|
286
|
+
# The fixture holds Julian 2024-01-01 through 2024-01-04. The same instants
|
|
287
|
+
# carry the Gregorian labels 2024-01-14 through 2024-01-17.
|
|
288
|
+
first = np.datetime64(arrays["time"]["stats"]["min"], "ns")
|
|
289
|
+
last = np.datetime64(arrays["time"]["stats"]["max"], "ns")
|
|
290
|
+
assert first == np.datetime64("2024-01-14T00:00:00")
|
|
291
|
+
assert last == np.datetime64("2024-01-17T00:00:00")
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def test_convert_calendar_matches_a_per_element_conversion(
|
|
295
|
+
netcdf_dir_with_a_julian_calendar,
|
|
296
|
+
):
|
|
297
|
+
"""The array-at-once conversion must equal the obvious slow one."""
|
|
298
|
+
from atlas.xarray import _cftime_to_datetime64
|
|
299
|
+
|
|
300
|
+
ds = xr.open_dataset(netcdf_dir_with_a_julian_calendar / "julian.nc")
|
|
301
|
+
values = np.asarray(ds["time"].values)
|
|
302
|
+
|
|
303
|
+
fast = _cftime_to_datetime64("time", values)
|
|
304
|
+
slow = np.array(
|
|
305
|
+
[
|
|
306
|
+
np.datetime64(v.change_calendar("proleptic_gregorian").isoformat(), "ns")
|
|
307
|
+
for v in values
|
|
308
|
+
]
|
|
309
|
+
)
|
|
310
|
+
assert np.array_equal(fast, slow)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def test_an_artificial_calendar_has_no_exact_gregorian_date(tmp_path):
|
|
314
|
+
"""A 360_day year names no real instant, so no conversion exists."""
|
|
315
|
+
import cftime
|
|
316
|
+
|
|
317
|
+
from atlas.xarray import _cftime_to_datetime64
|
|
318
|
+
|
|
319
|
+
values = np.array([cftime.Datetime360Day(2024, 2, 30)], dtype=object)
|
|
320
|
+
with pytest.raises(NotImplementedError, match="360_day"):
|
|
321
|
+
_cftime_to_datetime64("time", values)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def test_a_date_outside_the_nanosecond_range_is_refused_not_wrapped(tmp_path):
|
|
325
|
+
"""numpy wraps such a date in silence. Atlas must not pass one on."""
|
|
326
|
+
import cftime
|
|
327
|
+
|
|
328
|
+
from atlas.xarray import _cftime_to_datetime64
|
|
329
|
+
|
|
330
|
+
for year in (1600, 2300):
|
|
331
|
+
values = np.array([cftime.DatetimeJulian(year, 1, 1)], dtype=object)
|
|
332
|
+
with pytest.raises(NotImplementedError, match="outside the range"):
|
|
333
|
+
_cftime_to_datetime64("time", values)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def test_convert_calendar_leaves_a_normal_time_axis_alone(netcdf_dir, tmp_path):
|
|
337
|
+
dest = tmp_path / "c"
|
|
338
|
+
atlas.create(netcdf_dir, str(dest), convert_calendar=True)
|
|
339
|
+
arrays = {a["name"]: a for a in atlas.describe(str(dest), "2024-01.nc")["arrays"]}
|
|
340
|
+
assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def test_an_object_array_of_something_else_is_refused_clearly(tmp_path):
|
|
344
|
+
"""numpy reports `object` for both a string array and this one."""
|
|
345
|
+
from atlas.xarray import _reject_unstorable_object_array
|
|
346
|
+
|
|
347
|
+
var = xr.DataArray(np.array([{"a": 1}, {"b": 2}], dtype=object), dims=["x"])
|
|
348
|
+
with pytest.raises(NotImplementedError, match="object array of dict"):
|
|
349
|
+
_reject_unstorable_object_array("payload", var)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def test_a_real_string_array_still_passes_the_object_check(tmp_path):
|
|
353
|
+
from atlas.xarray import _reject_unstorable_object_array
|
|
354
|
+
|
|
355
|
+
# A missing cell must not be mistaken for an unsupported element.
|
|
356
|
+
var = xr.DataArray(np.array([None, "b", "c"], dtype=object), dims=["x"])
|
|
357
|
+
_reject_unstorable_object_array("label", var)
|
|
358
|
+
|
|
359
|
+
empty = xr.DataArray(np.array([], dtype=object), dims=["x"])
|
|
360
|
+
_reject_unstorable_object_array("empty", empty)
|
|
361
|
+
|
|
362
|
+
|
|
218
363
|
# ── the log file ─────────────────────────────────────────────────────
|
|
219
364
|
|
|
220
365
|
|
|
@@ -62,6 +62,8 @@ def create(
|
|
|
62
62
|
chunks: Optional[dict[str, Sequence[int]]] = None,
|
|
63
63
|
open_chunks: Union[str, dict[str, int], None] = "auto",
|
|
64
64
|
chunk_size: str = "128MiB",
|
|
65
|
+
decode_times: bool = True,
|
|
66
|
+
convert_calendar: bool = False,
|
|
65
67
|
on_error: str = "stop",
|
|
66
68
|
on_unsupported: str = "stop",
|
|
67
69
|
progress: Optional[Callable[[str], None]] = None,
|
|
@@ -114,6 +116,16 @@ def create(
|
|
|
114
116
|
chunk_size: The block size ``"auto"`` aims at, as a dask size string.
|
|
115
117
|
It is about the memory ceiling per variable during ingest. It
|
|
116
118
|
defaults to ``"128MiB"``.
|
|
119
|
+
decode_times: How xarray reads a time axis. Under the default, a
|
|
120
|
+
calendar it cannot map to ``datetime64[ns]``, such as a Julian
|
|
121
|
+
one, decodes to cftime objects, which atlas cannot store. Set it
|
|
122
|
+
false to keep the raw numbers, with their ``units`` and
|
|
123
|
+
``calendar`` attributes.
|
|
124
|
+
convert_calendar: Turn a cftime axis into exact Gregorian timestamps.
|
|
125
|
+
Each one keeps its instant, so a Julian ``1973-02-25`` becomes the
|
|
126
|
+
Gregorian ``1973-03-10`` that names the same moment. A calendar
|
|
127
|
+
with no real instant, such as ``360_day``, and a date outside the
|
|
128
|
+
nanosecond range both raise instead.
|
|
117
129
|
on_error: ``"stop"`` is the default. It abandons the whole collection
|
|
118
130
|
on the first bad file. ``"skip"`` records that file and continues.
|
|
119
131
|
on_unsupported: What one array of an unsupported dtype costs.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"""Runs the command as ``python -m atlas``.
|
|
2
2
|
|
|
3
3
|
The ``atlas`` console script needs its directory on ``PATH``. This entry point
|
|
4
|
-
does not. It works in any environment that can import the package
|
|
5
|
-
it the reliable form inside a container, a CI job,
|
|
6
|
-
nobody activated.
|
|
4
|
+
does not. It works in any environment that can import the package. That makes
|
|
5
|
+
it the reliable form inside a container, in a CI job, and in a virtual
|
|
6
|
+
environment nobody activated.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
import sys
|
|
@@ -161,6 +161,8 @@ def cmd_create(args: argparse.Namespace) -> int:
|
|
|
161
161
|
chunks=chunks,
|
|
162
162
|
open_chunks=open_chunks,
|
|
163
163
|
chunk_size=args.chunk_size,
|
|
164
|
+
decode_times=not args.no_decode_times,
|
|
165
|
+
convert_calendar=args.convert_calendar,
|
|
164
166
|
on_error="skip" if args.skip_errors else "stop",
|
|
165
167
|
on_unsupported="skip" if args.skip_unsupported else "stop",
|
|
166
168
|
progress=progress,
|
|
@@ -360,6 +362,23 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
360
362
|
action="store_true",
|
|
361
363
|
help="skip files that fail instead of abandoning the collection",
|
|
362
364
|
)
|
|
365
|
+
p.add_argument(
|
|
366
|
+
"--convert-calendar",
|
|
367
|
+
action="store_true",
|
|
368
|
+
help=(
|
|
369
|
+
"turn a cftime axis into exact Gregorian timestamps. Each one "
|
|
370
|
+
"keeps its instant, so a Julian date moves to the Gregorian date "
|
|
371
|
+
"that names the same moment"
|
|
372
|
+
),
|
|
373
|
+
)
|
|
374
|
+
p.add_argument(
|
|
375
|
+
"--no-decode-times",
|
|
376
|
+
action="store_true",
|
|
377
|
+
help=(
|
|
378
|
+
"keep a time axis as raw numbers. Use it for a calendar xarray "
|
|
379
|
+
"decodes to cftime, such as Julian, which atlas cannot store"
|
|
380
|
+
),
|
|
381
|
+
)
|
|
363
382
|
p.add_argument(
|
|
364
383
|
"--skip-unsupported",
|
|
365
384
|
action="store_true",
|
|
@@ -105,6 +105,8 @@ def create(
|
|
|
105
105
|
chunks: Optional[dict[str, Sequence[int]]] = None,
|
|
106
106
|
open_chunks: Any = "auto",
|
|
107
107
|
chunk_size: str = DEFAULT_CHUNK_SIZE,
|
|
108
|
+
decode_times: bool = True,
|
|
109
|
+
convert_calendar: bool = False,
|
|
108
110
|
on_error: str = "stop",
|
|
109
111
|
on_unsupported: str = "stop",
|
|
110
112
|
progress: Optional[Any] = None,
|
|
@@ -137,6 +139,17 @@ def create(
|
|
|
137
139
|
`on_error` then handles. `"skip"` leaves that array out, and the rest of
|
|
138
140
|
the dataset still lands.
|
|
139
141
|
|
|
142
|
+
`decode_times` controls how xarray reads a time axis. Under the default,
|
|
143
|
+
a calendar it cannot map to `datetime64[ns]`, such as a Julian one,
|
|
144
|
+
decodes to cftime objects, which atlas cannot store. Set it false to keep
|
|
145
|
+
the raw numbers and their `units` and `calendar` attributes instead.
|
|
146
|
+
|
|
147
|
+
`convert_calendar` turns those cftime objects into exact Gregorian
|
|
148
|
+
timestamps. Each one keeps its instant, so a Julian 1973-02-25 becomes the
|
|
149
|
+
Gregorian 1973-03-10 that names the same moment. A calendar with no real
|
|
150
|
+
instant, such as `360_day`, and a date outside the nanosecond range both
|
|
151
|
+
raise instead.
|
|
152
|
+
|
|
140
153
|
`progress` takes each file name as that file lands.
|
|
141
154
|
|
|
142
155
|
Returns a summary. How many datasets landed, which files the run skipped,
|
|
@@ -150,6 +163,8 @@ def create(
|
|
|
150
163
|
)
|
|
151
164
|
|
|
152
165
|
open_kwargs = _open_kwargs(open_chunks)
|
|
166
|
+
if not decode_times:
|
|
167
|
+
open_kwargs["decode_times"] = False
|
|
153
168
|
|
|
154
169
|
import dask
|
|
155
170
|
import xarray as xr
|
|
@@ -185,7 +200,13 @@ def create(
|
|
|
185
200
|
# before the file closes.
|
|
186
201
|
with xr.open_dataset(path, **open_kwargs) as ds:
|
|
187
202
|
left_out = _xarray._write_xarray_dataset(
|
|
188
|
-
writer,
|
|
203
|
+
writer,
|
|
204
|
+
ds,
|
|
205
|
+
name,
|
|
206
|
+
chunks,
|
|
207
|
+
None,
|
|
208
|
+
on_unsupported,
|
|
209
|
+
convert_calendar,
|
|
189
210
|
)
|
|
190
211
|
for item in left_out:
|
|
191
212
|
_LOG.warning(
|
|
@@ -244,6 +244,152 @@ def _fill_missing_strings(block: np.ndarray, fill: str) -> tuple[np.ndarray, int
|
|
|
244
244
|
return np.where(mask, fill, block), n
|
|
245
245
|
|
|
246
246
|
|
|
247
|
+
# How far into an object array to look for a real value. Enough to tell a
|
|
248
|
+
# string array from an array of something else, and cheap on a dask block.
|
|
249
|
+
_OBJECT_SAMPLE = 64
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _sample_object_element(var: Any) -> Any:
|
|
253
|
+
"""The first value of an object array that is no missing marker.
|
|
254
|
+
|
|
255
|
+
This scans a bounded prefix, and computes one dask block at most.
|
|
256
|
+
"""
|
|
257
|
+
data = var.data
|
|
258
|
+
if _is_dask_array(data):
|
|
259
|
+
import dask
|
|
260
|
+
|
|
261
|
+
(head,) = dask.compute(data.reshape(-1)[:_OBJECT_SAMPLE])
|
|
262
|
+
else:
|
|
263
|
+
head = np.asarray(data).reshape(-1)[:_OBJECT_SAMPLE]
|
|
264
|
+
for value in head:
|
|
265
|
+
if not _is_missing_str(value):
|
|
266
|
+
return value
|
|
267
|
+
return None
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
# The span of `datetime64[ns]`. A value outside it wraps silently, so every
|
|
271
|
+
# conversion is checked against these first.
|
|
272
|
+
_STAMP_MIN = np.datetime64("1677-09-21T00:12:44", "us")
|
|
273
|
+
_STAMP_MAX = np.datetime64("2262-04-11T23:47:16", "us")
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _is_cftime(value: Any) -> bool:
|
|
277
|
+
"""Whether `value` is a cftime datetime, without importing cftime."""
|
|
278
|
+
return type(value).__module__.split(".")[0] == "cftime"
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _cftime_to_datetime64(var_name: str, values: np.ndarray) -> np.ndarray:
|
|
282
|
+
"""The same instants as `values`, on the proleptic Gregorian calendar.
|
|
283
|
+
|
|
284
|
+
cftime maps between two real-world calendars through the Julian Day, so
|
|
285
|
+
the instant survives exactly. A Julian label and a Gregorian label of one
|
|
286
|
+
instant differ by 13 days this century, and name the same moment.
|
|
287
|
+
|
|
288
|
+
The conversion runs on the whole array at once. One call per element costs
|
|
289
|
+
a few hundred times more.
|
|
290
|
+
"""
|
|
291
|
+
import cftime
|
|
292
|
+
|
|
293
|
+
flat = np.ravel(values)
|
|
294
|
+
missing = np.fromiter(
|
|
295
|
+
(_is_missing_str(v) for v in flat), dtype=bool, count=flat.size
|
|
296
|
+
)
|
|
297
|
+
present = flat[~missing]
|
|
298
|
+
out = np.full(flat.size, np.datetime64("NaT", "us"), dtype="datetime64[us]")
|
|
299
|
+
if present.size == 0:
|
|
300
|
+
return out.reshape(values.shape).astype("datetime64[ns]")
|
|
301
|
+
|
|
302
|
+
calendar = present[0].calendar
|
|
303
|
+
try:
|
|
304
|
+
# The Unix epoch, written on the source calendar. Elapsed time from it
|
|
305
|
+
# is then elapsed time from the Gregorian epoch.
|
|
306
|
+
epoch = cftime.datetime(
|
|
307
|
+
1970, 1, 1, calendar="proleptic_gregorian"
|
|
308
|
+
).change_calendar(calendar)
|
|
309
|
+
except ValueError as exc:
|
|
310
|
+
raise NotImplementedError(
|
|
311
|
+
f"variable {var_name!r} uses the {calendar!r} calendar, which names "
|
|
312
|
+
f"no real instant, so no exact Gregorian date exists. Only a "
|
|
313
|
+
f"real-world calendar converts. Pass decode_times=False to keep "
|
|
314
|
+
f"the raw numbers"
|
|
315
|
+
) from exc
|
|
316
|
+
|
|
317
|
+
units = "microseconds since " + epoch.strftime("%Y-%m-%d %H:%M:%S")
|
|
318
|
+
micros = np.rint(
|
|
319
|
+
np.asarray(cftime.date2num(present, units, calendar=calendar), dtype="float64")
|
|
320
|
+
).astype("int64")
|
|
321
|
+
stamps = np.datetime64(0, "us") + micros.astype("timedelta64[us]")
|
|
322
|
+
|
|
323
|
+
outside = (stamps < _STAMP_MIN) | (stamps > _STAMP_MAX)
|
|
324
|
+
if outside.any():
|
|
325
|
+
worst = stamps[outside][0]
|
|
326
|
+
raise NotImplementedError(
|
|
327
|
+
f"variable {var_name!r} holds {int(outside.sum())} date(s) outside "
|
|
328
|
+
f"the range of a nanosecond timestamp, such as {worst}. Atlas "
|
|
329
|
+
f"stores {_STAMP_MIN} to {_STAMP_MAX}. Pass decode_times=False to "
|
|
330
|
+
f"keep the raw numbers"
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
out[~missing] = stamps
|
|
334
|
+
return out.reshape(values.shape).astype("datetime64[ns]")
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def with_converted_calendars(ds: "xr.Dataset") -> "xr.Dataset":
|
|
338
|
+
"""A copy of `ds` with every cftime array turned into `datetime64[ns]`.
|
|
339
|
+
|
|
340
|
+
Each timestamp keeps its instant. The calendar label changes, so a Julian
|
|
341
|
+
1973-02-25 becomes the Gregorian 1973-03-10 that names the same moment.
|
|
342
|
+
"""
|
|
343
|
+
replacements = {}
|
|
344
|
+
for name in list(ds.coords) + list(ds.data_vars):
|
|
345
|
+
var = ds[name]
|
|
346
|
+
if np.dtype(var.dtype).kind != "O":
|
|
347
|
+
continue
|
|
348
|
+
sample = _sample_object_element(var)
|
|
349
|
+
if sample is None or not _is_cftime(sample):
|
|
350
|
+
continue
|
|
351
|
+
values = _cftime_to_datetime64(str(name), np.asarray(var.values))
|
|
352
|
+
replacements[str(name)] = (var.dims, values, dict(var.attrs))
|
|
353
|
+
|
|
354
|
+
if not replacements:
|
|
355
|
+
return ds
|
|
356
|
+
out = ds.copy()
|
|
357
|
+
for name, entry in replacements.items():
|
|
358
|
+
# The (dims, data, attrs) tuple assigns without importing xarray here.
|
|
359
|
+
out[name] = entry
|
|
360
|
+
_LOG.info("converted %r from cftime to an exact Gregorian timestamp", name)
|
|
361
|
+
return out
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _reject_unstorable_object_array(var_name: str, var: Any) -> None:
|
|
365
|
+
"""Raises when an object array holds something atlas cannot store.
|
|
366
|
+
|
|
367
|
+
Atlas stores an object array as a string array. numpy reports only
|
|
368
|
+
`object` for the dtype, so the element type settles what the array really
|
|
369
|
+
holds. A missing marker and an empty array both pass, because the string
|
|
370
|
+
path handles them.
|
|
371
|
+
"""
|
|
372
|
+
sample = _sample_object_element(var)
|
|
373
|
+
if sample is None or isinstance(sample, (str, bytes)):
|
|
374
|
+
return
|
|
375
|
+
|
|
376
|
+
kind = type(sample)
|
|
377
|
+
if _is_cftime(sample):
|
|
378
|
+
raise NotImplementedError(
|
|
379
|
+
f"variable {var_name!r} holds cftime objects ({kind.__name__}), "
|
|
380
|
+
f"which atlas cannot store. xarray decodes a calendar it cannot "
|
|
381
|
+
f"map to datetime64[ns], such as a Julian one, into cftime. Pass "
|
|
382
|
+
f"convert_calendar=True, or --convert-calendar, for the exact "
|
|
383
|
+
f"Gregorian instant. Pass decode_times=False to keep the raw "
|
|
384
|
+
f"numbers and their units instead"
|
|
385
|
+
)
|
|
386
|
+
raise NotImplementedError(
|
|
387
|
+
f"variable {var_name!r} is an object array of {kind.__name__}. Atlas "
|
|
388
|
+
f"stores an object array as a string, so every element must be a str "
|
|
389
|
+
f"or bytes"
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
|
|
247
393
|
def _is_dask_array(arr: Any) -> bool:
|
|
248
394
|
"""True when `arr` is a `dask.array.Array`. False when dask is absent."""
|
|
249
395
|
try:
|
|
@@ -372,6 +518,7 @@ def _write_xarray_to_view(
|
|
|
372
518
|
chunks: Optional[dict[str, Sequence[int]]] = None,
|
|
373
519
|
fill_value: Any = None,
|
|
374
520
|
on_unsupported: str = "stop",
|
|
521
|
+
convert_calendar: bool = False,
|
|
375
522
|
) -> list[dict[str, str]]:
|
|
376
523
|
"""Fills an empty `DatasetWriter` from an xarray Dataset.
|
|
377
524
|
|
|
@@ -389,6 +536,9 @@ def _write_xarray_to_view(
|
|
|
389
536
|
f"on_unsupported must be 'stop' or 'skip', got {on_unsupported!r}"
|
|
390
537
|
)
|
|
391
538
|
|
|
539
|
+
if convert_calendar:
|
|
540
|
+
ds = with_converted_calendars(ds)
|
|
541
|
+
|
|
392
542
|
coord_names = [str(n) for n in ds.coords.keys()]
|
|
393
543
|
order = coord_names + [str(n) for n in ds.data_vars.keys()]
|
|
394
544
|
|
|
@@ -399,7 +549,14 @@ def _write_xarray_to_view(
|
|
|
399
549
|
skipped: list[dict[str, str]] = []
|
|
400
550
|
for var_name in order:
|
|
401
551
|
try:
|
|
402
|
-
|
|
552
|
+
np_dtype = np.dtype(ds[var_name].dtype)
|
|
553
|
+
atlas_dtype = _np_to_atlas_dtype(np_dtype)
|
|
554
|
+
# An object array maps to string on the dtype alone. Check what it
|
|
555
|
+
# really holds before any array is defined, so a skip leaves no
|
|
556
|
+
# half-written array behind.
|
|
557
|
+
if np_dtype.kind == "O":
|
|
558
|
+
_reject_unstorable_object_array(var_name, ds[var_name])
|
|
559
|
+
dtypes[var_name] = atlas_dtype
|
|
403
560
|
except NotImplementedError as exc:
|
|
404
561
|
if on_unsupported == "stop":
|
|
405
562
|
raise
|
|
@@ -525,6 +682,7 @@ def _write_xarray_dataset(
|
|
|
525
682
|
chunks: Optional[dict[str, Sequence[int]]] = None,
|
|
526
683
|
fill_value: Any = None,
|
|
527
684
|
on_unsupported: str = "stop",
|
|
685
|
+
convert_calendar: bool = False,
|
|
528
686
|
) -> list[dict[str, str]]:
|
|
529
687
|
"""Writes an ``xarray.Dataset`` into an open writer, under ``name``.
|
|
530
688
|
|
|
@@ -543,6 +701,7 @@ def _write_xarray_dataset(
|
|
|
543
701
|
chunks=chunks,
|
|
544
702
|
fill_value=fill_value,
|
|
545
703
|
on_unsupported=on_unsupported,
|
|
704
|
+
convert_calendar=convert_calendar,
|
|
546
705
|
)
|
|
547
706
|
except BaseException:
|
|
548
707
|
dataset_writer.abort()
|
|
index a385694..dbe79f1 100644
|
|
|
Binary file
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|