atlas-python 0.16.2__tar.gz → 0.16.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {atlas_python-0.16.2 → atlas_python-0.16.3}/Cargo.lock +2 -2
  2. {atlas_python-0.16.2 → atlas_python-0.16.3}/Cargo.toml +1 -1
  3. {atlas_python-0.16.2 → atlas_python-0.16.3}/PKG-INFO +1 -1
  4. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/Cargo.toml +1 -1
  5. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/cli.md +2 -1
  6. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/dtypes.md +65 -0
  7. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/writer.rs +1 -1
  8. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_ops.py +145 -0
  9. {atlas_python-0.16.2 → atlas_python-0.16.3}/pyproject.toml +1 -1
  10. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__init__.pyi +12 -0
  11. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__main__.py +3 -3
  12. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_cli.py +19 -0
  13. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_ops.py +22 -1
  14. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/xarray.py +160 -1
  15. {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/fixtures/from_python/data.atlas +0 -0
  16. {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-python-docs.yaml +0 -0
  17. {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-python-release.yaml +0 -0
  18. {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/atlas-rust-release.yaml +0 -0
  19. {atlas_python-0.16.2 → atlas_python-0.16.3}/.github/workflows/ci.yaml +0 -0
  20. {atlas_python-0.16.2 → atlas_python-0.16.3}/.gitignore +0 -0
  21. {atlas_python-0.16.2 → atlas_python-0.16.3}/CONTRIBUTING.md +0 -0
  22. {atlas_python-0.16.2 → atlas_python-0.16.3}/LICENSE +0 -0
  23. {atlas_python-0.16.2 → atlas_python-0.16.3}/README.md +0 -0
  24. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/.python-version +0 -0
  25. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/LICENSE +0 -0
  26. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/README.md +0 -0
  27. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/examples.md +0 -0
  28. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/cloud-storage.md +0 -0
  29. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/creating.md +0 -0
  30. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/inspecting.md +0 -0
  31. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/reading-data.md +0 -0
  32. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/guides/removing.md +0 -0
  33. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/index.md +0 -0
  34. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/installation.md +0 -0
  35. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/quickstart.md +0 -0
  36. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/reference/api.md +0 -0
  37. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/docs/vs-zarr-netcdf.md +0 -0
  38. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/examples/01_library.py +0 -0
  39. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/examples/02_object_store.py +0 -0
  40. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/mkdocs.yml +0 -0
  41. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/attr.rs +0 -0
  42. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/dtype.rs +0 -0
  43. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/error.rs +0 -0
  44. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/lib.rs +0 -0
  45. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/logging.rs +0 -0
  46. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/reader.rs +0 -0
  47. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/runtime.rs +0 -0
  48. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/src/source.rs +0 -0
  49. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
  50. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/conftest.py +0 -0
  51. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/make_fixture.py +0 -0
  52. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_cli.py +0 -0
  53. {atlas_python-0.16.2 → atlas_python-0.16.3}/atlas-python/tests/test_source.py +0 -0
  54. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/README.md +0 -0
  55. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/architecture.md +0 -0
  56. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/data-model.md +0 -0
  57. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/format.md +0 -0
  58. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/python.md +0 -0
  59. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/read-path.md +0 -0
  60. {atlas_python-0.16.2 → atlas_python-0.16.3}/docs/write-path.md +0 -0
  61. {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/lifecycle.rs +0 -0
  62. {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/sensor_fleet.rs +0 -0
  63. {atlas_python-0.16.2 → atlas_python-0.16.3}/examples/weather_store.rs +0 -0
  64. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/__init__.py +0 -0
  65. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_log.py +0 -0
  66. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/_source.py +0 -0
  67. {atlas_python-0.16.2 → atlas_python-0.16.3}/python/atlas/py.typed +0 -0
  68. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/config.rs +0 -0
  69. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/error.rs +0 -0
  70. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/footer.rs +0 -0
  71. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/mask.rs +0 -0
  72. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/mod.rs +0 -0
  73. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/format/segment_store.rs +0 -0
  74. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/lib.rs +0 -0
  75. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/reader/mod.rs +0 -0
  76. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/array.rs +0 -0
  77. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/attr.rs +0 -0
  78. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/dtype.rs +0 -0
  79. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/schema/mod.rs +0 -0
  80. {atlas_python-0.16.2 → atlas_python-0.16.3}/src/writer/mod.rs +0 -0
  81. {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/cross_fixture.rs +0 -0
  82. {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/fixtures/golden_v1/data.atlas +0 -0
  83. {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/golden.rs +0 -0
  84. {atlas_python-0.16.2 → atlas_python-0.16.3}/tests/integration.rs +0 -0
@@ -71,7 +71,7 @@ dependencies = [
71
71
 
72
72
  [[package]]
73
73
  name = "atlas-python"
74
- version = "0.16.2"
74
+ version = "0.16.3"
75
75
  dependencies = [
76
76
  "atlas-rust",
77
77
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
86
86
 
87
87
  [[package]]
88
88
  name = "atlas-rust"
89
- version = "0.16.2"
89
+ version = "0.16.3"
90
90
  dependencies = [
91
91
  "array-format",
92
92
  "async-trait",
@@ -5,7 +5,7 @@ members = ["atlas-python"]
5
5
  # Registry name on crates.io (`atlas` is taken). The library is still
6
6
  # imported as `atlas` — see `[lib]` below.
7
7
  name = "atlas-rust"
8
- version = "0.16.2"
8
+ version = "0.16.3"
9
9
  edition = "2024"
10
10
  description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
11
11
  license = "Apache-2.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: atlas-python
3
- Version: 0.16.2
3
+ Version: 0.16.3
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Science/Research
6
6
  Classifier: Programming Language :: Python :: 3
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "atlas-python"
3
- version = "0.16.2"
3
+ version = "0.16.3"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -30,7 +30,7 @@ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
30
30
  ```bash
31
31
  $ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
32
32
  $ cat ingest.log
33
- 2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.2: create /data/nc ...
33
+ 2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.3: create /data/nc ...
34
34
  2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
35
35
  2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
36
36
  2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
@@ -66,6 +66,7 @@ A failure part-way leaves no collection, and not a partial one.
66
66
  | `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
67
67
  | `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
68
68
  | `--skip-errors` | Skip files that fail instead of abandoning the collection |
69
+ | `--no-decode-times` | Keep a time axis as raw numbers, for a calendar that decodes to cftime |
69
70
  | `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
70
71
  | `-q`, `--quiet` | Do not list a file as it lands |
71
72
 
@@ -17,6 +17,71 @@ What a NetCDF variable becomes when `atlas create` ingests it.
17
17
  `atlas show` prints the atlas name, so a `datetime64[ns]` variable appears as
18
18
  `timestamp_nanoseconds`.
19
19
 
20
+ ### Calendars that decode to cftime
21
+
22
+ xarray decodes a time axis to `datetime64[ns]` only when the calendar allows
23
+ it. A Julian, `360_day`, or `noleap` calendar, or a date outside the
24
+ `datetime64[ns]` range, decodes to a `cftime` object instead. Those arrive as
25
+ a numpy `object` array, and atlas cannot store one:
26
+
27
+ ```text
28
+ atlas: profile.nc: variable 'JULD' holds cftime objects (DatetimeJulian),
29
+ which atlas cannot store...
30
+ ```
31
+
32
+ Atlas refuses by default rather than guess, because two conversions exist and
33
+ they differ by 13 days. Pick one.
34
+
35
+ **Convert to the exact instant.** This is almost always what you want:
36
+
37
+ ```bash
38
+ atlas create /data/nc /data/collection --convert-calendar
39
+ ```
40
+
41
+ ```python
42
+ atlas.create("/data/nc", dest, convert_calendar=True)
43
+ ```
44
+
45
+ The axis becomes `timestamp_nanoseconds`, and every value keeps the moment it
46
+ named. A Julian `1973-02-25 18:15` becomes the Gregorian `1973-03-10 18:15`.
47
+ Those are one instant under two calendars. cftime maps between them through
48
+ the Julian Day, so nothing is approximated.
49
+
50
+ **Keep the raw numbers.** The axis stores as an integer, with its `units` and
51
+ `calendar` attributes beside it. A reader decodes it later with
52
+ `cftime.num2date`:
53
+
54
+ ```bash
55
+ atlas create /data/nc /data/collection --no-decode-times
56
+ ```
57
+
58
+ **Drop the axis.** `--skip-unsupported` leaves the time array out, and keeps
59
+ the rest of the dataset.
60
+
61
+ #### Two conversions, 13 days apart
62
+
63
+ The Julian calendar takes a leap year every four years. The Gregorian calendar
64
+ drops three of those every four centuries. The gap grows by about three days
65
+ per 400 years, and holds at 13 days from 1900 to 2100.
66
+
67
+ | Conversion | Julian 2024-01-01 becomes | It keeps |
68
+ |---|---|---|
69
+ | `--convert-calendar` | 2024-01-14 | the instant |
70
+ | `xr.Dataset.convert_calendar("standard")` | 2024-01-01 | the labels |
71
+
72
+ The xarray method keeps the year, month, and day, which moves the moment by 13
73
+ days. Use it only when the labels matter more than the instant.
74
+
75
+ #### Two limits
76
+
77
+ **An artificial calendar names no instant.** A `360_day`, `noleap`, or
78
+ `all_leap` year is a model convention with no place on a real timeline.
79
+ `--convert-calendar` raises for one. Use `--no-decode-times` instead.
80
+
81
+ **A nanosecond timestamp spans 1677-09-21 to 2262-04-11.** A date outside that
82
+ raises. numpy wraps such a date in silence, so atlas checks every value before
83
+ it stores one.
84
+
20
85
  ### datetime and timedelta
21
86
 
22
87
  Atlas supports the `[ns]` resolution of `datetime64` alone. It rejects every
@@ -322,7 +322,7 @@ impl PyDatasetWriter {
322
322
  String::from_utf8_lossy(&b).into_owned()
323
323
  } else {
324
324
  return Err(PyTypeError::new_err(format!(
325
- "string array element must be str or bytes, got {:?}",
325
+ "array {name:?}: a string element must be str or bytes, got {:?}",
326
326
  bound.get_type().name()?
327
327
  )));
328
328
  };
@@ -215,6 +215,151 @@ def test_a_clean_ingest_reports_no_skipped_arrays(netcdf_dir, tmp_path):
215
215
  assert result["skipped_arrays"] == []
216
216
 
217
217
 
218
+ # ── cftime calendars ─────────────────────────────────────────────────
219
+
220
+
221
+ @pytest.fixture
222
+ def netcdf_dir_with_a_julian_calendar(tmp_path):
223
+ """A time axis xarray decodes to cftime, not to datetime64."""
224
+ import cftime
225
+
226
+ d = tmp_path / "nc"
227
+ d.mkdir()
228
+ times = [cftime.DatetimeJulian(2024, 1, i + 1) for i in range(4)]
229
+ xr.Dataset(
230
+ {"temp": ("time", np.arange(4, dtype=np.float32))},
231
+ coords={"time": ("time", times)},
232
+ ).to_netcdf(d / "julian.nc")
233
+ return d
234
+
235
+
236
+ def test_a_cftime_axis_reports_the_variable_and_the_remedy(
237
+ netcdf_dir_with_a_julian_calendar, tmp_path
238
+ ):
239
+ with pytest.raises(atlas.AtlasError) as exc:
240
+ atlas.create(netcdf_dir_with_a_julian_calendar, str(tmp_path / "c"))
241
+
242
+ message = str(exc.value)
243
+ assert "'time'" in message, "the message must name the variable"
244
+ assert "DatetimeJulian" in message
245
+ assert "decode_times=False" in message
246
+
247
+
248
+ def test_decode_times_false_stores_a_cftime_axis_as_raw_numbers(
249
+ netcdf_dir_with_a_julian_calendar, tmp_path
250
+ ):
251
+ dest = tmp_path / "c"
252
+ atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), decode_times=False)
253
+
254
+ arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
255
+ assert arrays["time"]["dtype"] == "int64"
256
+ # The units and the calendar survive, so a reader can decode the numbers.
257
+ assert arrays["time"]["attributes"]["calendar"] == "julian"
258
+ assert "days since" in arrays["time"]["attributes"]["units"]
259
+ assert arrays["temp"]["dtype"] == "float32"
260
+
261
+
262
+ def test_a_cftime_axis_can_be_skipped_per_array(
263
+ netcdf_dir_with_a_julian_calendar, tmp_path
264
+ ):
265
+ dest = tmp_path / "c"
266
+ result = atlas.create(
267
+ netcdf_dir_with_a_julian_calendar, str(dest), on_unsupported="skip"
268
+ )
269
+
270
+ assert [s["array"] for s in result["skipped_arrays"]] == ["time"]
271
+ assert [a["name"] for a in atlas.describe(str(dest), "julian.nc")["arrays"]] == [
272
+ "temp"
273
+ ]
274
+
275
+
276
+ def test_convert_calendar_keeps_the_exact_instant(
277
+ netcdf_dir_with_a_julian_calendar, tmp_path
278
+ ):
279
+ """A Julian label and its Gregorian label name one moment, 13 days apart."""
280
+ dest = tmp_path / "c"
281
+ atlas.create(netcdf_dir_with_a_julian_calendar, str(dest), convert_calendar=True)
282
+
283
+ arrays = {a["name"]: a for a in atlas.describe(str(dest), "julian.nc")["arrays"]}
284
+ assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
285
+
286
+ # The fixture holds Julian 2024-01-01 through 2024-01-04. The same instants
287
+ # carry the Gregorian labels 2024-01-14 through 2024-01-17.
288
+ first = np.datetime64(arrays["time"]["stats"]["min"], "ns")
289
+ last = np.datetime64(arrays["time"]["stats"]["max"], "ns")
290
+ assert first == np.datetime64("2024-01-14T00:00:00")
291
+ assert last == np.datetime64("2024-01-17T00:00:00")
292
+
293
+
294
+ def test_convert_calendar_matches_a_per_element_conversion(
295
+ netcdf_dir_with_a_julian_calendar,
296
+ ):
297
+ """The array-at-once conversion must equal the obvious slow one."""
298
+ from atlas.xarray import _cftime_to_datetime64
299
+
300
+ ds = xr.open_dataset(netcdf_dir_with_a_julian_calendar / "julian.nc")
301
+ values = np.asarray(ds["time"].values)
302
+
303
+ fast = _cftime_to_datetime64("time", values)
304
+ slow = np.array(
305
+ [
306
+ np.datetime64(v.change_calendar("proleptic_gregorian").isoformat(), "ns")
307
+ for v in values
308
+ ]
309
+ )
310
+ assert np.array_equal(fast, slow)
311
+
312
+
313
+ def test_an_artificial_calendar_has_no_exact_gregorian_date(tmp_path):
314
+ """A 360_day year names no real instant, so no conversion exists."""
315
+ import cftime
316
+
317
+ from atlas.xarray import _cftime_to_datetime64
318
+
319
+ values = np.array([cftime.Datetime360Day(2024, 2, 30)], dtype=object)
320
+ with pytest.raises(NotImplementedError, match="360_day"):
321
+ _cftime_to_datetime64("time", values)
322
+
323
+
324
+ def test_a_date_outside_the_nanosecond_range_is_refused_not_wrapped(tmp_path):
325
+ """numpy wraps such a date in silence. Atlas must not pass one on."""
326
+ import cftime
327
+
328
+ from atlas.xarray import _cftime_to_datetime64
329
+
330
+ for year in (1600, 2300):
331
+ values = np.array([cftime.DatetimeJulian(year, 1, 1)], dtype=object)
332
+ with pytest.raises(NotImplementedError, match="outside the range"):
333
+ _cftime_to_datetime64("time", values)
334
+
335
+
336
+ def test_convert_calendar_leaves_a_normal_time_axis_alone(netcdf_dir, tmp_path):
337
+ dest = tmp_path / "c"
338
+ atlas.create(netcdf_dir, str(dest), convert_calendar=True)
339
+ arrays = {a["name"]: a for a in atlas.describe(str(dest), "2024-01.nc")["arrays"]}
340
+ assert arrays["time"]["dtype"] == "timestamp_nanoseconds"
341
+
342
+
343
+ def test_an_object_array_of_something_else_is_refused_clearly(tmp_path):
344
+ """numpy reports `object` for both a string array and this one."""
345
+ from atlas.xarray import _reject_unstorable_object_array
346
+
347
+ var = xr.DataArray(np.array([{"a": 1}, {"b": 2}], dtype=object), dims=["x"])
348
+ with pytest.raises(NotImplementedError, match="object array of dict"):
349
+ _reject_unstorable_object_array("payload", var)
350
+
351
+
352
+ def test_a_real_string_array_still_passes_the_object_check(tmp_path):
353
+ from atlas.xarray import _reject_unstorable_object_array
354
+
355
+ # A missing cell must not be mistaken for an unsupported element.
356
+ var = xr.DataArray(np.array([None, "b", "c"], dtype=object), dims=["x"])
357
+ _reject_unstorable_object_array("label", var)
358
+
359
+ empty = xr.DataArray(np.array([], dtype=object), dims=["x"])
360
+ _reject_unstorable_object_array("empty", empty)
361
+
362
+
218
363
  # ── the log file ─────────────────────────────────────────────────────
219
364
 
220
365
 
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "atlas-python"
7
- version = "0.16.2"
7
+ version = "0.16.3"
8
8
  description = "Python bindings for the ATLAS array store"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -62,6 +62,8 @@ def create(
62
62
  chunks: Optional[dict[str, Sequence[int]]] = None,
63
63
  open_chunks: Union[str, dict[str, int], None] = "auto",
64
64
  chunk_size: str = "128MiB",
65
+ decode_times: bool = True,
66
+ convert_calendar: bool = False,
65
67
  on_error: str = "stop",
66
68
  on_unsupported: str = "stop",
67
69
  progress: Optional[Callable[[str], None]] = None,
@@ -114,6 +116,16 @@ def create(
114
116
  chunk_size: The block size ``"auto"`` aims at, as a dask size string.
115
117
  It is about the memory ceiling per variable during ingest. It
116
118
  defaults to ``"128MiB"``.
119
+ decode_times: How xarray reads a time axis. Under the default, a
120
+ calendar it cannot map to ``datetime64[ns]``, such as a Julian
121
+ one, decodes to cftime objects, which atlas cannot store. Set it
122
+ false to keep the raw numbers, with their ``units`` and
123
+ ``calendar`` attributes.
124
+ convert_calendar: Turn a cftime axis into exact Gregorian timestamps.
125
+ Each one keeps its instant, so a Julian ``1973-02-25`` becomes the
126
+ Gregorian ``1973-03-10`` that names the same moment. A calendar
127
+ with no real instant, such as ``360_day``, and a date outside the
128
+ nanosecond range both raise instead.
117
129
  on_error: ``"stop"`` is the default. It abandons the whole collection
118
130
  on the first bad file. ``"skip"`` records that file and continues.
119
131
  on_unsupported: What one array of an unsupported dtype costs.
@@ -1,9 +1,9 @@
1
1
  """Runs the command as ``python -m atlas``.
2
2
 
3
3
  The ``atlas`` console script needs its directory on ``PATH``. This entry point
4
- does not. It works in any environment that can import the package, which makes
5
- it the reliable form inside a container, a CI job, or a virtual environment
6
- nobody activated.
4
+ does not. It works in any environment that can import the package. That makes
5
+ it the reliable form inside a container, in a CI job, and in a virtual
6
+ environment nobody activated.
7
7
  """
8
8
 
9
9
  import sys
@@ -161,6 +161,8 @@ def cmd_create(args: argparse.Namespace) -> int:
161
161
  chunks=chunks,
162
162
  open_chunks=open_chunks,
163
163
  chunk_size=args.chunk_size,
164
+ decode_times=not args.no_decode_times,
165
+ convert_calendar=args.convert_calendar,
164
166
  on_error="skip" if args.skip_errors else "stop",
165
167
  on_unsupported="skip" if args.skip_unsupported else "stop",
166
168
  progress=progress,
@@ -360,6 +362,23 @@ def build_parser() -> argparse.ArgumentParser:
360
362
  action="store_true",
361
363
  help="skip files that fail instead of abandoning the collection",
362
364
  )
365
+ p.add_argument(
366
+ "--convert-calendar",
367
+ action="store_true",
368
+ help=(
369
+ "turn a cftime axis into exact Gregorian timestamps. Each one "
370
+ "keeps its instant, so a Julian date moves to the Gregorian date "
371
+ "that names the same moment"
372
+ ),
373
+ )
374
+ p.add_argument(
375
+ "--no-decode-times",
376
+ action="store_true",
377
+ help=(
378
+ "keep a time axis as raw numbers. Use it for a calendar xarray "
379
+ "decodes to cftime, such as Julian, which atlas cannot store"
380
+ ),
381
+ )
363
382
  p.add_argument(
364
383
  "--skip-unsupported",
365
384
  action="store_true",
@@ -105,6 +105,8 @@ def create(
105
105
  chunks: Optional[dict[str, Sequence[int]]] = None,
106
106
  open_chunks: Any = "auto",
107
107
  chunk_size: str = DEFAULT_CHUNK_SIZE,
108
+ decode_times: bool = True,
109
+ convert_calendar: bool = False,
108
110
  on_error: str = "stop",
109
111
  on_unsupported: str = "stop",
110
112
  progress: Optional[Any] = None,
@@ -137,6 +139,17 @@ def create(
137
139
  `on_error` then handles. `"skip"` leaves that array out, and the rest of
138
140
  the dataset still lands.
139
141
 
142
+ `decode_times` controls how xarray reads a time axis. Under the default,
143
+ a calendar it cannot map to `datetime64[ns]`, such as a Julian one,
144
+ decodes to cftime objects, which atlas cannot store. Set it false to keep
145
+ the raw numbers and their `units` and `calendar` attributes instead.
146
+
147
+ `convert_calendar` turns those cftime objects into exact Gregorian
148
+ timestamps. Each one keeps its instant, so a Julian 1973-02-25 becomes the
149
+ Gregorian 1973-03-10 that names the same moment. A calendar with no real
150
+ instant, such as `360_day`, and a date outside the nanosecond range both
151
+ raise instead.
152
+
140
153
  `progress` takes each file name as that file lands.
141
154
 
142
155
  Returns a summary. How many datasets landed, which files the run skipped,
@@ -150,6 +163,8 @@ def create(
150
163
  )
151
164
 
152
165
  open_kwargs = _open_kwargs(open_chunks)
166
+ if not decode_times:
167
+ open_kwargs["decode_times"] = False
153
168
 
154
169
  import dask
155
170
  import xarray as xr
@@ -185,7 +200,13 @@ def create(
185
200
  # before the file closes.
186
201
  with xr.open_dataset(path, **open_kwargs) as ds:
187
202
  left_out = _xarray._write_xarray_dataset(
188
- writer, ds, name, chunks, None, on_unsupported
203
+ writer,
204
+ ds,
205
+ name,
206
+ chunks,
207
+ None,
208
+ on_unsupported,
209
+ convert_calendar,
189
210
  )
190
211
  for item in left_out:
191
212
  _LOG.warning(
@@ -244,6 +244,152 @@ def _fill_missing_strings(block: np.ndarray, fill: str) -> tuple[np.ndarray, int
244
244
  return np.where(mask, fill, block), n
245
245
 
246
246
 
247
+ # How far into an object array to look for a real value. Enough to tell a
248
+ # string array from an array of something else, and cheap on a dask block.
249
+ _OBJECT_SAMPLE = 64
250
+
251
+
252
+ def _sample_object_element(var: Any) -> Any:
253
+ """The first value of an object array that is no missing marker.
254
+
255
+ This scans a bounded prefix, and computes one dask block at most.
256
+ """
257
+ data = var.data
258
+ if _is_dask_array(data):
259
+ import dask
260
+
261
+ (head,) = dask.compute(data.reshape(-1)[:_OBJECT_SAMPLE])
262
+ else:
263
+ head = np.asarray(data).reshape(-1)[:_OBJECT_SAMPLE]
264
+ for value in head:
265
+ if not _is_missing_str(value):
266
+ return value
267
+ return None
268
+
269
+
270
+ # The span of `datetime64[ns]`. A value outside it wraps silently, so every
271
+ # conversion is checked against these first.
272
+ _STAMP_MIN = np.datetime64("1677-09-21T00:12:44", "us")
273
+ _STAMP_MAX = np.datetime64("2262-04-11T23:47:16", "us")
274
+
275
+
276
+ def _is_cftime(value: Any) -> bool:
277
+ """Whether `value` is a cftime datetime, without importing cftime."""
278
+ return type(value).__module__.split(".")[0] == "cftime"
279
+
280
+
281
+ def _cftime_to_datetime64(var_name: str, values: np.ndarray) -> np.ndarray:
282
+ """The same instants as `values`, on the proleptic Gregorian calendar.
283
+
284
+ cftime maps between two real-world calendars through the Julian Day, so
285
+ the instant survives exactly. A Julian label and a Gregorian label of one
286
+ instant differ by 13 days this century, and name the same moment.
287
+
288
+ The conversion runs on the whole array at once. One call per element costs
289
+ a few hundred times more.
290
+ """
291
+ import cftime
292
+
293
+ flat = np.ravel(values)
294
+ missing = np.fromiter(
295
+ (_is_missing_str(v) for v in flat), dtype=bool, count=flat.size
296
+ )
297
+ present = flat[~missing]
298
+ out = np.full(flat.size, np.datetime64("NaT", "us"), dtype="datetime64[us]")
299
+ if present.size == 0:
300
+ return out.reshape(values.shape).astype("datetime64[ns]")
301
+
302
+ calendar = present[0].calendar
303
+ try:
304
+ # The Unix epoch, written on the source calendar. Elapsed time from it
305
+ # is then elapsed time from the Gregorian epoch.
306
+ epoch = cftime.datetime(
307
+ 1970, 1, 1, calendar="proleptic_gregorian"
308
+ ).change_calendar(calendar)
309
+ except ValueError as exc:
310
+ raise NotImplementedError(
311
+ f"variable {var_name!r} uses the {calendar!r} calendar, which names "
312
+ f"no real instant, so no exact Gregorian date exists. Only a "
313
+ f"real-world calendar converts. Pass decode_times=False to keep "
314
+ f"the raw numbers"
315
+ ) from exc
316
+
317
+ units = "microseconds since " + epoch.strftime("%Y-%m-%d %H:%M:%S")
318
+ micros = np.rint(
319
+ np.asarray(cftime.date2num(present, units, calendar=calendar), dtype="float64")
320
+ ).astype("int64")
321
+ stamps = np.datetime64(0, "us") + micros.astype("timedelta64[us]")
322
+
323
+ outside = (stamps < _STAMP_MIN) | (stamps > _STAMP_MAX)
324
+ if outside.any():
325
+ worst = stamps[outside][0]
326
+ raise NotImplementedError(
327
+ f"variable {var_name!r} holds {int(outside.sum())} date(s) outside "
328
+ f"the range of a nanosecond timestamp, such as {worst}. Atlas "
329
+ f"stores {_STAMP_MIN} to {_STAMP_MAX}. Pass decode_times=False to "
330
+ f"keep the raw numbers"
331
+ )
332
+
333
+ out[~missing] = stamps
334
+ return out.reshape(values.shape).astype("datetime64[ns]")
335
+
336
+
337
+ def with_converted_calendars(ds: "xr.Dataset") -> "xr.Dataset":
338
+ """A copy of `ds` with every cftime array turned into `datetime64[ns]`.
339
+
340
+ Each timestamp keeps its instant. The calendar label changes, so a Julian
341
+ 1973-02-25 becomes the Gregorian 1973-03-10 that names the same moment.
342
+ """
343
+ replacements = {}
344
+ for name in list(ds.coords) + list(ds.data_vars):
345
+ var = ds[name]
346
+ if np.dtype(var.dtype).kind != "O":
347
+ continue
348
+ sample = _sample_object_element(var)
349
+ if sample is None or not _is_cftime(sample):
350
+ continue
351
+ values = _cftime_to_datetime64(str(name), np.asarray(var.values))
352
+ replacements[str(name)] = (var.dims, values, dict(var.attrs))
353
+
354
+ if not replacements:
355
+ return ds
356
+ out = ds.copy()
357
+ for name, entry in replacements.items():
358
+ # The (dims, data, attrs) tuple assigns without importing xarray here.
359
+ out[name] = entry
360
+ _LOG.info("converted %r from cftime to an exact Gregorian timestamp", name)
361
+ return out
362
+
363
+
364
+ def _reject_unstorable_object_array(var_name: str, var: Any) -> None:
365
+ """Raises when an object array holds something atlas cannot store.
366
+
367
+ Atlas stores an object array as a string array. numpy reports only
368
+ `object` for the dtype, so the element type settles what the array really
369
+ holds. A missing marker and an empty array both pass, because the string
370
+ path handles them.
371
+ """
372
+ sample = _sample_object_element(var)
373
+ if sample is None or isinstance(sample, (str, bytes)):
374
+ return
375
+
376
+ kind = type(sample)
377
+ if _is_cftime(sample):
378
+ raise NotImplementedError(
379
+ f"variable {var_name!r} holds cftime objects ({kind.__name__}), "
380
+ f"which atlas cannot store. xarray decodes a calendar it cannot "
381
+ f"map to datetime64[ns], such as a Julian one, into cftime. Pass "
382
+ f"convert_calendar=True, or --convert-calendar, for the exact "
383
+ f"Gregorian instant. Pass decode_times=False to keep the raw "
384
+ f"numbers and their units instead"
385
+ )
386
+ raise NotImplementedError(
387
+ f"variable {var_name!r} is an object array of {kind.__name__}. Atlas "
388
+ f"stores an object array as a string, so every element must be a str "
389
+ f"or bytes"
390
+ )
391
+
392
+
247
393
  def _is_dask_array(arr: Any) -> bool:
248
394
  """True when `arr` is a `dask.array.Array`. False when dask is absent."""
249
395
  try:
@@ -372,6 +518,7 @@ def _write_xarray_to_view(
372
518
  chunks: Optional[dict[str, Sequence[int]]] = None,
373
519
  fill_value: Any = None,
374
520
  on_unsupported: str = "stop",
521
+ convert_calendar: bool = False,
375
522
  ) -> list[dict[str, str]]:
376
523
  """Fills an empty `DatasetWriter` from an xarray Dataset.
377
524
 
@@ -389,6 +536,9 @@ def _write_xarray_to_view(
389
536
  f"on_unsupported must be 'stop' or 'skip', got {on_unsupported!r}"
390
537
  )
391
538
 
539
+ if convert_calendar:
540
+ ds = with_converted_calendars(ds)
541
+
392
542
  coord_names = [str(n) for n in ds.coords.keys()]
393
543
  order = coord_names + [str(n) for n in ds.data_vars.keys()]
394
544
 
@@ -399,7 +549,14 @@ def _write_xarray_to_view(
399
549
  skipped: list[dict[str, str]] = []
400
550
  for var_name in order:
401
551
  try:
402
- dtypes[var_name] = _np_to_atlas_dtype(np.dtype(ds[var_name].dtype))
552
+ np_dtype = np.dtype(ds[var_name].dtype)
553
+ atlas_dtype = _np_to_atlas_dtype(np_dtype)
554
+ # An object array maps to string on the dtype alone. Check what it
555
+ # really holds before any array is defined, so a skip leaves no
556
+ # half-written array behind.
557
+ if np_dtype.kind == "O":
558
+ _reject_unstorable_object_array(var_name, ds[var_name])
559
+ dtypes[var_name] = atlas_dtype
403
560
  except NotImplementedError as exc:
404
561
  if on_unsupported == "stop":
405
562
  raise
@@ -525,6 +682,7 @@ def _write_xarray_dataset(
525
682
  chunks: Optional[dict[str, Sequence[int]]] = None,
526
683
  fill_value: Any = None,
527
684
  on_unsupported: str = "stop",
685
+ convert_calendar: bool = False,
528
686
  ) -> list[dict[str, str]]:
529
687
  """Writes an ``xarray.Dataset`` into an open writer, under ``name``.
530
688
 
@@ -543,6 +701,7 @@ def _write_xarray_dataset(
543
701
  chunks=chunks,
544
702
  fill_value=fill_value,
545
703
  on_unsupported=on_unsupported,
704
+ convert_calendar=convert_calendar,
546
705
  )
547
706
  except BaseException:
548
707
  dataset_writer.abort()
index a385694..dbe79f1 100644
Binary file
File without changes
File without changes
File without changes
File without changes