atlas-python 0.16.0__tar.gz → 0.16.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {atlas_python-0.16.0 → atlas_python-0.16.1}/Cargo.lock +2 -2
  2. {atlas_python-0.16.0 → atlas_python-0.16.1}/Cargo.toml +1 -1
  3. {atlas_python-0.16.0 → atlas_python-0.16.1}/PKG-INFO +16 -9
  4. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/Cargo.toml +1 -1
  5. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/README.md +15 -8
  6. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/cli.md +53 -11
  7. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/examples.md +2 -2
  8. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/cloud-storage.md +2 -2
  9. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/creating.md +83 -9
  10. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/dtypes.md +6 -3
  11. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/inspecting.md +5 -5
  12. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/reading-data.md +4 -4
  13. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/removing.md +5 -5
  14. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/index.md +5 -5
  15. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/quickstart.md +13 -13
  16. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/reference/api.md +4 -0
  17. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/vs-zarr-netcdf.md +1 -1
  18. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/examples/01_library.py +4 -4
  19. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/examples/02_object_store.py +2 -2
  20. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/conftest.py +20 -1
  21. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_cli.py +70 -17
  22. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_ops.py +176 -47
  23. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_source.py +3 -3
  24. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/format.md +10 -3
  25. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/python.md +29 -4
  26. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/write-path.md +1 -0
  27. {atlas_python-0.16.0 → atlas_python-0.16.1}/pyproject.toml +1 -1
  28. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/__init__.py +2 -0
  29. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/__init__.pyi +44 -9
  30. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_cli.py +42 -8
  31. atlas_python-0.16.1/python/atlas/_log.py +82 -0
  32. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_ops.py +60 -14
  33. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/xarray.py +64 -11
  34. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/footer.rs +39 -3
  35. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/writer/mod.rs +10 -0
  36. {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/cross_fixture.rs +6 -6
  37. {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/integration.rs +25 -0
  38. {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-python-docs.yaml +0 -0
  39. {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-python-release.yaml +0 -0
  40. {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-rust-release.yaml +0 -0
  41. {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/ci.yaml +0 -0
  42. {atlas_python-0.16.0 → atlas_python-0.16.1}/.gitignore +0 -0
  43. {atlas_python-0.16.0 → atlas_python-0.16.1}/CONTRIBUTING.md +0 -0
  44. {atlas_python-0.16.0 → atlas_python-0.16.1}/LICENSE +0 -0
  45. {atlas_python-0.16.0 → atlas_python-0.16.1}/README.md +0 -0
  46. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/.python-version +0 -0
  47. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/LICENSE +0 -0
  48. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/installation.md +0 -0
  49. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/mkdocs.yml +0 -0
  50. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/attr.rs +0 -0
  51. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/dtype.rs +0 -0
  52. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/error.rs +0 -0
  53. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/lib.rs +0 -0
  54. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/logging.rs +0 -0
  55. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/reader.rs +0 -0
  56. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/runtime.rs +0 -0
  57. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/source.rs +0 -0
  58. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/writer.rs +0 -0
  59. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
  60. {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/make_fixture.py +0 -0
  61. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/README.md +0 -0
  62. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/architecture.md +0 -0
  63. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/data-model.md +0 -0
  64. {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/read-path.md +0 -0
  65. {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/lifecycle.rs +0 -0
  66. {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/sensor_fleet.rs +0 -0
  67. {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/weather_store.rs +0 -0
  68. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_source.py +0 -0
  69. {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/py.typed +0 -0
  70. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/config.rs +0 -0
  71. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/error.rs +0 -0
  72. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/mask.rs +0 -0
  73. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/mod.rs +0 -0
  74. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/segment_store.rs +0 -0
  75. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/lib.rs +0 -0
  76. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/reader/mod.rs +0 -0
  77. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/array.rs +0 -0
  78. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/attr.rs +0 -0
  79. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/dtype.rs +0 -0
  80. {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/mod.rs +0 -0
  81. {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/fixtures/from_python/data.atlas +0 -0
  82. {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/fixtures/golden_v1/data.atlas +0 -0
  83. {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/golden.rs +0 -0
@@ -71,7 +71,7 @@ dependencies = [
71
71
 
72
72
  [[package]]
73
73
  name = "atlas-python"
74
- version = "0.16.0"
74
+ version = "0.16.1"
75
75
  dependencies = [
76
76
  "atlas-rust",
77
77
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
86
86
 
87
87
  [[package]]
88
88
  name = "atlas-rust"
89
- version = "0.16.0"
89
+ version = "0.16.1"
90
90
  dependencies = [
91
91
  "array-format",
92
92
  "async-trait",
@@ -5,7 +5,7 @@ members = ["atlas-python"]
5
5
  # Registry name on crates.io (`atlas` is taken). The library is still
6
6
  # imported as `atlas` — see `[lib]` below.
7
7
  name = "atlas-rust"
8
- version = "0.16.0"
8
+ version = "0.16.1"
9
9
  edition = "2024"
10
10
  description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
11
11
  license = "Apache-2.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: atlas-python
3
- Version: 0.16.0
3
+ Version: 0.16.1
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Science/Research
6
6
  Classifier: Programming Language :: Python :: 3
@@ -39,9 +39,9 @@ pip install atlas-python
39
39
  ```bash
40
40
  atlas create /data/nc /data/collection
41
41
  atlas ls /data/collection
42
- atlas show /data/collection 2024-01
42
+ atlas show /data/collection 2024-01.nc
43
43
  atlas info /data/collection
44
- atlas rm /data/collection 2024-02 2024-03
44
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
45
45
  ```
46
46
 
47
47
  | Extra | Install | Adds |
@@ -58,10 +58,10 @@ The same five as a library:
58
58
  import atlas
59
59
 
60
60
  atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
61
- atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
62
- atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
61
+ atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
62
+ atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
63
63
  atlas.info("/data/collection") # counts, size, codec, statistics
64
- atlas.remove("/data/collection", ["2024-02"]) # updates the mask
64
+ atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
65
65
  ```
66
66
 
67
67
  Every one takes a local path, a URL, or an obstore handle:
@@ -94,8 +94,8 @@ request.
94
94
  ## What `show` gives you
95
95
 
96
96
  ```text
97
- $ atlas show /data/collection 2024-01
98
- dataset 2024-01 {
97
+ $ atlas show /data/collection 2024-01.nc
98
+ dataset 2024-01.nc {
99
99
  dimensions:
100
100
  lat = 4 ;
101
101
  lon = 6 ;
@@ -121,7 +121,7 @@ command gives the same content as a structure.
121
121
  ## Ingest
122
122
 
123
123
  `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
124
- them, and writes one dataset per file, named after the stem. Each coordinate
124
+ them, and writes one dataset per file, named after the file. Each coordinate
125
125
  and data variable becomes an array. Each variable attribute becomes a per-array
126
126
  attribute. `_FillValue` becomes the fill of the array.
127
127
 
@@ -134,6 +134,13 @@ Nothing at the destination is readable until every file lands. A failure
134
134
  part-way leaves no collection, and not a partial one. `on_error="skip"`, or
135
135
  `--skip-errors`, trades that for progress.
136
136
 
137
+ Atlas cannot store every numpy dtype, and `bool` is the common case. One such
138
+ variable fails the whole file by default. `on_unsupported="skip"`, or
139
+ `--skip-unsupported`, leaves out that one array and lands the rest.
140
+
141
+ `--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
142
+ warning to a file, each with its reason and the file it came from.
143
+
137
144
  ## dtypes
138
145
 
139
146
  | numpy | atlas |
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "atlas-python"
3
- version = "0.16.0"
3
+ version = "0.16.1"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -11,9 +11,9 @@ pip install atlas-python
11
11
  ```bash
12
12
  atlas create /data/nc /data/collection
13
13
  atlas ls /data/collection
14
- atlas show /data/collection 2024-01
14
+ atlas show /data/collection 2024-01.nc
15
15
  atlas info /data/collection
16
- atlas rm /data/collection 2024-02 2024-03
16
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
17
17
  ```
18
18
 
19
19
  | Extra | Install | Adds |
@@ -30,10 +30,10 @@ The same five as a library:
30
30
  import atlas
31
31
 
32
32
  atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
33
- atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
34
- atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
33
+ atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
34
+ atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
35
35
  atlas.info("/data/collection") # counts, size, codec, statistics
36
- atlas.remove("/data/collection", ["2024-02"]) # updates the mask
36
+ atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
37
37
  ```
38
38
 
39
39
  Every one takes a local path, a URL, or an obstore handle:
@@ -66,8 +66,8 @@ request.
66
66
  ## What `show` gives you
67
67
 
68
68
  ```text
69
- $ atlas show /data/collection 2024-01
70
- dataset 2024-01 {
69
+ $ atlas show /data/collection 2024-01.nc
70
+ dataset 2024-01.nc {
71
71
  dimensions:
72
72
  lat = 4 ;
73
73
  lon = 6 ;
@@ -93,7 +93,7 @@ command gives the same content as a structure.
93
93
  ## Ingest
94
94
 
95
95
  `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
96
- them, and writes one dataset per file, named after the stem. Each coordinate
96
+ them, and writes one dataset per file, named after the file. Each coordinate
97
97
  and data variable becomes an array. Each variable attribute becomes a per-array
98
98
  attribute. `_FillValue` becomes the fill of the array.
99
99
 
@@ -106,6 +106,13 @@ Nothing at the destination is readable until every file lands. A failure
106
106
  part-way leaves no collection, and not a partial one. `on_error="skip"`, or
107
107
  `--skip-errors`, trades that for progress.
108
108
 
109
+ Atlas cannot store every numpy dtype, and `bool` is the common case. One such
110
+ variable fails the whole file by default. `on_unsupported="skip"`, or
111
+ `--skip-unsupported`, leaves out that one array and lands the rest.
112
+
113
+ `--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
114
+ warning to a file, each with its reason and the file it came from.
115
+
109
116
  ## dtypes
110
117
 
111
118
  | numpy | atlas |
@@ -16,8 +16,27 @@ A `<collection>` is a local path or a URL: `s3://bucket/prefix`, `gs://...`,
16
16
  `pip install "atlas-python[cloud]"`. See
17
17
  [Cloud storage](guides/cloud-storage.md).
18
18
 
19
- Every subcommand takes `--json`, and the remote flags `--region`,
20
- `--endpoint`, and `--anonymous`.
19
+ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
20
+ `--region`, `--endpoint`, and `--anonymous`.
21
+
22
+ ## Logging
23
+
24
+ `--log-file PATH` appends every error and warning to a file, with the reason:
25
+
26
+ ```bash
27
+ $ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
28
+ $ cat ingest.log
29
+ 2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.1: create /data/nc ...
30
+ 2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
31
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
32
+ 2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
33
+ ```
34
+
35
+ The file opens in append mode. Each line names the file it came from, so a
36
+ large ingest stays readable.
37
+
38
+ This is separate from `ATLAS_LOG`, which turns on the Rust `tracing` stream to
39
+ stderr.
21
40
 
22
41
  ## create
23
42
 
@@ -25,9 +44,9 @@ Every subcommand takes `--json`, and the remote flags `--region`,
25
44
  atlas create /data/nc /data/collection
26
45
  ```
27
46
 
28
- Each NetCDF file becomes one dataset, named after its stem. `2024-01.nc`
29
- becomes `2024-01`. The files land in sorted order, which makes the ordinals of
30
- a collection reproducible.
47
+ Each NetCDF file becomes one dataset, named after the file. `2024-01.nc`
48
+ becomes `2024-01.nc`, suffix and all. The files land in sorted order, which
49
+ makes the ordinals of a collection reproducible.
31
50
 
32
51
  Nothing at the destination is readable until every file lands, with the footer.
33
52
  A failure part-way leaves no collection, and not a partial one.
@@ -40,10 +59,33 @@ A failure part-way leaves no collection, and not a partial one.
40
59
  | `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
41
60
  | `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
42
61
  | `--skip-errors` | Skip files that fail instead of abandoning the collection |
62
+ | `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
43
63
  | `-q`, `--quiet` | Do not list a file as it lands |
44
64
 
45
65
  Progress goes to stderr, so a pipe still reads stdout.
46
66
 
67
+ ### Unsupported dtypes
68
+
69
+ Atlas cannot store every numpy dtype. `bool` is the common case. By default one
70
+ such variable fails the whole file:
71
+
72
+ ```bash
73
+ $ atlas create /data/nc /data/collection
74
+ atlas: /data/nc/buoy.nc: numpy dtype dtype('bool') is not supported by atlas ...
75
+ ```
76
+
77
+ `--skip-unsupported` narrows that to the one array:
78
+
79
+ ```bash
80
+ $ atlas create /data/nc /data/collection --skip-unsupported
81
+ skipped array buoy/flag (bool)
82
+ 1 dataset(s) written to /data/collection
83
+ ```
84
+
85
+ Every other array of that dataset lands, with its attributes. `--json` reports
86
+ the skipped arrays under `skipped_arrays`, with the dataset, the dtype, and the
87
+ reason. See [Supported dtypes](guides/dtypes.md).
88
+
47
89
  ### Large files
48
90
 
49
91
  Each file reads in dask blocks. A file far larger than memory therefore
@@ -88,7 +130,7 @@ atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
88
130
  ## rm
89
131
 
90
132
  ```bash
91
- atlas rm /data/collection 2024-02 2024-03
133
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
92
134
  ```
93
135
 
94
136
  This removes several datasets in one call. A name is a dataset name, or the
@@ -114,9 +156,9 @@ error.
114
156
 
115
157
  ```bash
116
158
  $ atlas ls /data/collection
117
- 2024-01
118
- 2024-02
119
- 2024-03
159
+ 2024-01.nc
160
+ 2024-02.nc
161
+ 2024-03.nc
120
162
  ```
121
163
 
122
164
  One name per line, in write order. A removed dataset does not appear. This
@@ -131,8 +173,8 @@ atlas ls /data/collection --json | jq . # as a JSON array
131
173
  ## show
132
174
 
133
175
  ```bash
134
- $ atlas show /data/collection 2024-01
135
- dataset 2024-01 {
176
+ $ atlas show /data/collection 2024-01.nc
177
+ dataset 2024-01.nc {
136
178
  dimensions:
137
179
  lat = 4 ;
138
180
  lon = 6 ;
@@ -19,8 +19,8 @@ python atlas-python/examples/01_library.py
19
19
  atlas create /data/nc /data/collection
20
20
  atlas ls /data/collection
21
21
  atlas info /data/collection
22
- atlas show /data/collection 2024-01
23
- atlas rm /data/collection 2024-02 2024-03
22
+ atlas show /data/collection 2024-01.nc
23
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
24
24
  ```
25
25
 
26
26
  See [The `atlas` command](cli.md).
@@ -51,8 +51,8 @@ store = obs.store.S3Store(
51
51
 
52
52
  atlas.create("/data/nc", store)
53
53
  atlas.list_datasets(store)
54
- atlas.describe(store, "2024-01")
55
- atlas.remove(store, ["2024-02"])
54
+ atlas.describe(store, "2024-01.nc")
55
+ atlas.remove(store, ["2024-02.nc"])
56
56
  atlas.info(store)
57
57
  ```
58
58
 
@@ -15,8 +15,9 @@ atlas create /data/nc /data/collection
15
15
  ## What happens
16
16
 
17
17
  `create` collects every file that matches `.nc`, `.nc4`, `.cdf`, or `.netcdf`.
18
- It **sorts** them, and writes one dataset per file, named after the file stem.
19
- `2024-01.nc` becomes the dataset `2024-01`.
18
+ It **sorts** them, and writes one dataset per file, named after the file.
19
+ `2024-01.nc` becomes the dataset `2024-01.nc`. The suffix is part of the name,
20
+ so `jan.nc` and `jan.nc4` are two datasets.
20
21
 
21
22
  The sort matters. An ordinal comes from the write order, so a sorted ingest
22
23
  makes it reproducible. Rebuild the same directory, and every dataset lands at
@@ -39,7 +40,7 @@ That is what you usually want. When it is not:
39
40
 
40
41
  ```python
41
42
  result = atlas.create("/data/nc", dest, on_error="skip")
42
- result["written"] # ['2024-01', '2024-03']
43
+ result["written"] # ['2024-01.nc', '2024-03.nc']
43
44
  result["skipped"] # [{'file': '.../2024-02.nc', 'error': '...'}]
44
45
  ```
45
46
 
@@ -51,6 +52,78 @@ A skipped file leaves no trace in the collection. The writer moves to the next
51
52
  one. The CLI exits `1` when it skipped anything, so a pipeline sees that. It
52
53
  still writes the collection.
53
54
 
55
+ ## One bad array, not one bad file
56
+
57
+ `on_error` works at the granularity of a file. One variable of an unsupported
58
+ dtype therefore costs the whole dataset. `on_unsupported="skip"` narrows that
59
+ to the array:
60
+
61
+ ```python
62
+ result = atlas.create("/data/nc", dest, on_unsupported="skip")
63
+ result["skipped_arrays"]
64
+ # [{'array': 'flag', 'dtype': 'bool', 'error': '...', 'dataset': '2024-01.nc'}]
65
+ ```
66
+
67
+ ```bash
68
+ atlas create /data/nc /data/collection --skip-unsupported
69
+ ```
70
+
71
+ The rest of the dataset lands as usual: every other array, every attribute,
72
+ and the dataset itself. The skipped name is absent from the schema, so no
73
+ empty array stands in for it.
74
+
75
+ Atlas resolves every dtype before it defines the first array. A skip therefore
76
+ never leaves a half-written array behind.
77
+
78
+ The two settings compose. `--skip-unsupported` handles the array atlas cannot
79
+ store. `--skip-errors` handles the file that fails for any other reason.
80
+
81
+ See [Supported dtypes](dtypes.md) for what atlas can store.
82
+
83
+ ## The log file
84
+
85
+ Both kinds of skip go to a log file, with the reason:
86
+
87
+ ```bash
88
+ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
89
+ ```
90
+
91
+ ```text
92
+ 2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
93
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/broken.nc: ValueError: did not find a match ...
94
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
95
+ 2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 1 file(s) and 1 array(s)
96
+ ```
97
+
98
+ Each line names the file, so a thousand-file ingest stays readable. The file
99
+ opens in append mode, so a repeat run adds to it.
100
+
101
+ From the library:
102
+
103
+ ```python
104
+ import atlas
105
+
106
+ atlas.log_to_file("ingest.log")
107
+ atlas.create("/data/nc", dest, on_unsupported="skip")
108
+ ```
109
+
110
+ Atlas logs to the `atlas` logger and attaches no handler of its own, so attach
111
+ your own to send the records somewhere else:
112
+
113
+ ```python
114
+ import logging
115
+
116
+ logging.getLogger("atlas").addHandler(logging.StreamHandler())
117
+ logging.getLogger("atlas").setLevel(logging.INFO)
118
+ ```
119
+
120
+ `log_to_file` also captures Python warnings, such as the one about missing
121
+ string cells. That moves them off stderr, because `logging.captureWarnings` is
122
+ process-wide.
123
+
124
+ The Rust core logs separately, through `tracing`. `atlas.init_tracing()` sends
125
+ that stream to stderr. See [Installation](../installation.md).
126
+
54
127
  ## Chunking and memory
55
128
 
56
129
  These are one decision. The blocks a file *reads* in are the chunks it
@@ -114,12 +187,12 @@ alone. A one-chunk array reads whole, or not at all.
114
187
  Confirm what landed:
115
188
 
116
189
  ```python
117
- arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01")["arrays"]}
190
+ arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01.nc")["arrays"]}
118
191
  arrays["temperature"]["chunk_shape"]
119
192
  ```
120
193
 
121
194
  ```bash
122
- atlas show dest 2024-01 | grep _ChunkShape
195
+ atlas show dest 2024-01.nc | grep _ChunkShape
123
196
  ```
124
197
 
125
198
  ### The writer's own memory
@@ -157,12 +230,13 @@ turns it off.
157
230
  | Situation | Result |
158
231
  |---|---|
159
232
  | Directory holds no NetCDF files | `AtlasError` |
160
- | Two files share a stem | `AtlasError`. A dataset name must be unique |
161
- | A variable has a dtype atlas cannot store | `AtlasError` (or skipped) |
233
+ | Two files share a name | `AtlasError`. A dataset name must be unique |
234
+ | A variable has a dtype atlas cannot store | `AtlasError`, or one skipped array under `--skip-unsupported` |
235
+ | Any other bad file | `AtlasError`, or one skipped file under `--skip-errors` |
162
236
  | Destination URL cannot be resolved | `SourceError` |
163
237
 
164
- Two files with one stem surprise people. `a/jan.nc` and `b/jan.nc` both want
165
- the name `jan`. Rename one, or ingest them into two collections.
238
+ Two files with one name surprise people. `a/jan.nc` and `b/jan.nc` both want
239
+ the name `jan.nc`. Rename one, or ingest them into two collections.
166
240
 
167
241
  For the dtype rules, see [Supported dtypes](dtypes.md).
168
242
 
@@ -62,7 +62,7 @@ fill of the array, and not as an attribute.
62
62
  Check what landed:
63
63
 
64
64
  ```python
65
- arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01")["arrays"]}
65
+ arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01.nc")["arrays"]}
66
66
  arrays["temperature"]["fill_value"]
67
67
  ```
68
68
 
@@ -86,7 +86,10 @@ array data, not to the attributes. They come through unchanged:
86
86
  | `list[...]`, `fixed_size_list[..., N]` | Not exposed yet |
87
87
 
88
88
  A NetCDF file with a boolean variable fails the ingest. Store it as `uint8`,
89
- and write the convention down.
89
+ and write the convention down. To keep the rest of that file, pass
90
+ `--skip-unsupported`, or `on_unsupported="skip"`. That leaves out the one array
91
+ and lands everything else. See
92
+ [Creating a collection](creating.md#one-bad-array-not-one-bad-file).
90
93
 
91
94
  All four work as an **attribute** value, which is where they usually appear.
92
95
 
@@ -100,7 +103,7 @@ Some xarray attributes are no scalar. A nested dict, a ragged list, and a numpy
100
103
  array each encode as JSON on the way in. `describe` decodes them again:
101
104
 
102
105
  ```python
103
- atlas.describe(collection, "2024-01")["attributes"]
106
+ atlas.describe(collection, "2024-01.nc")["attributes"]
104
107
  # {'month': 1, 'bounds': [1.0, 2.0], 'nested': {'a': 1}}
105
108
  ```
106
109
 
@@ -7,7 +7,7 @@ fetches array data, so a catalogue of ten thousand datasets costs one request.
7
7
 
8
8
  ```python
9
9
  atlas.list_datasets("/data/collection")
10
- # ['2024-01', '2024-02', '2024-03']
10
+ # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
11
11
  ```
12
12
 
13
13
  ```bash
@@ -63,11 +63,11 @@ Use `describe` for the statistics of one dataset on its own.
63
63
  ## One dataset in detail: `describe` and `show`
64
64
 
65
65
  ```python
66
- atlas.describe("/data/collection", "2024-01")
66
+ atlas.describe("/data/collection", "2024-01.nc")
67
67
  ```
68
68
 
69
69
  ```bash
70
- atlas show /data/collection 2024-01
70
+ atlas show /data/collection 2024-01.nc
71
71
  ```
72
72
 
73
73
  The CLI prints it like `ncdump -h`. The library returns the structure. Both
@@ -78,7 +78,7 @@ attributes, the coordinate flag, and the statistics.
78
78
  `name` is a dataset name, or the NetCDF path the dataset came from:
79
79
 
80
80
  ```python
81
- atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01"
81
+ atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01.nc"
82
82
  ```
83
83
 
84
84
  ### Statistics
@@ -117,7 +117,7 @@ The ingest records which variables were xarray coordinates, and reports them
117
117
  back:
118
118
 
119
119
  ```python
120
- detail = atlas.describe(collection, "2024-01")
120
+ detail = atlas.describe(collection, "2024-01.nc")
121
121
  detail["coordinates"] # ['lat', 'lon']
122
122
  [a["name"] for a in detail["arrays"] if a["is_coordinate"]]
123
123
  ```
@@ -4,10 +4,10 @@ Python reads the **metadata** of a collection. It reads no array value.
4
4
 
5
5
  ```python
6
6
  atlas.list_datasets(collection) # ✓
7
- atlas.describe(collection, "jan") # ✓ types, shapes, attrs, stats
7
+ atlas.describe(collection, "jan.nc") # ✓ types, shapes, attrs, stats
8
8
  atlas.info(collection) # ✓ counts, size, collection stats
9
9
 
10
- atlas.read_array(collection, "jan", "t") # ✗ does not exist
10
+ atlas.read_array(collection, "jan.nc", "t") # ✗ does not exist
11
11
  ```
12
12
 
13
13
  ## Why
@@ -27,7 +27,7 @@ It is enough to build a catalogue, to check an ingest, or to choose the
27
27
  datasets worth a fetch:
28
28
 
29
29
  ```python
30
- detail = atlas.describe(collection, "2024-01")
30
+ detail = atlas.describe(collection, "2024-01.nc")
31
31
 
32
32
  detail["dimensions"] # {'lat': 4, 'lon': 6}
33
33
  detail["coordinates"] # ['lat', 'lon']
@@ -99,7 +99,7 @@ simpler than anything else here.
99
99
  reader opens it:
100
100
 
101
101
  ```python
102
- start, end = atlas.describe(collection, "2024-01")["segment_range"]
102
+ start, end = atlas.describe(collection, "2024-01.nc")["segment_range"]
103
103
  blob = open(f"{collection}/data.atlas", "rb").read()[start:end]
104
104
  open("2024-01.af", "wb").write(blob)
105
105
  ```
@@ -1,11 +1,11 @@
1
1
  # Removing datasets
2
2
 
3
3
  ```python
4
- atlas.remove("/data/collection", ["2024-02", "2024-03"])
4
+ atlas.remove("/data/collection", ["2024-02.nc", "2024-03.nc"])
5
5
  ```
6
6
 
7
7
  ```bash
8
- atlas rm /data/collection 2024-02 2024-03
8
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
9
9
  ```
10
10
 
11
11
  One call, whatever the number of datasets. A name is a dataset name, or the
@@ -66,7 +66,7 @@ mask holds a sorted set of ordinals.
66
66
  The same holds on the command line, up to the argument limit of your shell:
67
67
 
68
68
  ```bash
69
- atlas rm /data/collection 2024-01 2024-02 2024-03
69
+ atlas rm /data/collection 2024-01.nc 2024-02.nc 2024-03.nc
70
70
  ```
71
71
 
72
72
  For a list too long for one command line, call `atlas.remove` from Python.
@@ -83,8 +83,8 @@ atlas.remove(collection, ["nope"])
83
83
  `missing_ok` reports it instead:
84
84
 
85
85
  ```python
86
- result = atlas.remove(collection, ["2024-01", "nope"], missing_ok=True)
87
- result["removed"] # ['2024-01']
86
+ result = atlas.remove(collection, ["2024-01.nc", "nope"], missing_ok=True)
87
+ result["removed"] # ['2024-01.nc']
88
88
  result["missing"] # ['nope']
89
89
  ```
90
90
 
@@ -10,9 +10,9 @@ atlas create /data/nc /data/collection
10
10
 
11
11
  ```text
12
12
  $ atlas ls /data/collection
13
- 2024-01
14
- 2024-02
15
- 2024-03
13
+ 2024-01.nc
14
+ 2024-02.nc
15
+ 2024-03.nc
16
16
  ```
17
17
 
18
18
  ## The shape of it
@@ -60,8 +60,8 @@ atlas.list_datasets("s3://bucket/2024", region="eu-west-1")
60
60
  ## What `show` gives you
61
61
 
62
62
  ```text
63
- $ atlas show /data/collection 2024-01
64
- dataset 2024-01 {
63
+ $ atlas show /data/collection 2024-01.nc
64
+ dataset 2024-01.nc {
65
65
  dimensions:
66
66
  lat = 4 ;
67
67
  lon = 6 ;
@@ -10,13 +10,13 @@ atlas create /data/nc /data/collection
10
10
 
11
11
  ```text
12
12
  Writing /data/collection
13
- 2024-01
14
- 2024-02
15
- 2024-03
13
+ 2024-01.nc
14
+ 2024-02.nc
15
+ 2024-03.nc
16
16
  3 dataset(s) written to /data/collection
17
17
  ```
18
18
 
19
- Each file becomes one dataset, named after its stem. The result is one file:
19
+ Each file becomes one dataset, named after the file. The result is one file:
20
20
 
21
21
  ```bash
22
22
  $ ls /data/collection
@@ -38,9 +38,9 @@ atlas.create("/data/nc", "/data/collection")
38
38
 
39
39
  ```bash
40
40
  $ atlas ls /data/collection
41
- 2024-01
42
- 2024-02
43
- 2024-03
41
+ 2024-01.nc
42
+ 2024-02.nc
43
+ 2024-03.nc
44
44
  ```
45
45
 
46
46
  ```bash
@@ -72,8 +72,8 @@ The maximum comes from March. Removed datasets do not count.
72
72
  ## Inspect one dataset
73
73
 
74
74
  ```bash
75
- $ atlas show /data/collection 2024-01
76
- dataset 2024-01 {
75
+ $ atlas show /data/collection 2024-01.nc
76
+ dataset 2024-01.nc {
77
77
  dimensions:
78
78
  lat = 4 ;
79
79
  lon = 6 ;
@@ -103,7 +103,7 @@ footer holds those too, so to print them costs nothing extra.
103
103
  From Python, the same thing as a structure:
104
104
 
105
105
  ```python
106
- detail = atlas.describe("/data/collection", "2024-01")
106
+ detail = atlas.describe("/data/collection", "2024-01.nc")
107
107
  detail["dimensions"] # {'lat': 4, 'lon': 6}
108
108
  detail["coordinates"] # ['lat', 'lon']
109
109
  {a["name"]: a["stats"] for a in detail["arrays"]}
@@ -112,8 +112,8 @@ detail["coordinates"] # ['lat', 'lon']
112
112
  ## Remove
113
113
 
114
114
  ```bash
115
- $ atlas rm /data/collection 2024-02
116
- removed 1: 2024-02
115
+ $ atlas rm /data/collection 2024-02.nc
116
+ removed 1: 2024-02.nc
117
117
  2 dataset(s) remain
118
118
  ```
119
119
 
@@ -139,7 +139,7 @@ Not from Python. The Rust API reads array values:
139
139
 
140
140
  ```rust
141
141
  let atlas = Atlas::open_path("/data/collection").await?;
142
- let ds = atlas.dataset("2024-01")?;
142
+ let ds = atlas.dataset("2024-01.nc")?;
143
143
  let window = ds.read_array::<f32>("temperature", vec![0, 0], vec![2, 3]).await?;
144
144
  ```
145
145
 
@@ -35,6 +35,10 @@ import atlas
35
35
  options:
36
36
  heading_level: 3
37
37
 
38
+ ::: atlas.log_to_file
39
+ options:
40
+ heading_level: 3
41
+
38
42
  ::: atlas.init_tracing
39
43
  options:
40
44
  heading_level: 3