atlas-python 0.16.0__tar.gz → 0.16.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atlas_python-0.16.0 → atlas_python-0.16.1}/Cargo.lock +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.1}/Cargo.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.1}/PKG-INFO +16 -9
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/Cargo.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/README.md +15 -8
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/cli.md +53 -11
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/examples.md +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/cloud-storage.md +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/creating.md +83 -9
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/dtypes.md +6 -3
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/inspecting.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/reading-data.md +4 -4
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/guides/removing.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/index.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/quickstart.md +13 -13
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/reference/api.md +4 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/vs-zarr-netcdf.md +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/examples/01_library.py +4 -4
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/examples/02_object_store.py +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/conftest.py +20 -1
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_cli.py +70 -17
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_ops.py +176 -47
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/test_source.py +3 -3
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/format.md +10 -3
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/python.md +29 -4
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/write-path.md +1 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/pyproject.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/__init__.py +2 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/__init__.pyi +44 -9
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_cli.py +42 -8
- atlas_python-0.16.1/python/atlas/_log.py +82 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_ops.py +60 -14
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/xarray.py +64 -11
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/footer.rs +39 -3
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/writer/mod.rs +10 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/cross_fixture.rs +6 -6
- {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/integration.rs +25 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-python-docs.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-python-release.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/atlas-rust-release.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/.github/workflows/ci.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/.gitignore +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/CONTRIBUTING.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/LICENSE +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/README.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/.python-version +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/LICENSE +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/docs/installation.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/mkdocs.yml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/attr.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/dtype.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/error.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/lib.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/logging.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/reader.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/runtime.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/source.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/src/writer.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/atlas-python/tests/make_fixture.py +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/README.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/architecture.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/data-model.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/docs/read-path.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/lifecycle.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/sensor_fleet.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/examples/weather_store.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/_source.py +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/python/atlas/py.typed +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/config.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/error.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/mask.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/format/segment_store.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/lib.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/reader/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/array.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/attr.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/dtype.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/src/schema/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/fixtures/from_python/data.atlas +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/fixtures/golden_v1/data.atlas +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.1}/tests/golden.rs +0 -0
|
@@ -71,7 +71,7 @@ dependencies = [
|
|
|
71
71
|
|
|
72
72
|
[[package]]
|
|
73
73
|
name = "atlas-python"
|
|
74
|
-
version = "0.16.
|
|
74
|
+
version = "0.16.1"
|
|
75
75
|
dependencies = [
|
|
76
76
|
"atlas-rust",
|
|
77
77
|
"ndarray",
|
|
@@ -86,7 +86,7 @@ dependencies = [
|
|
|
86
86
|
|
|
87
87
|
[[package]]
|
|
88
88
|
name = "atlas-rust"
|
|
89
|
-
version = "0.16.
|
|
89
|
+
version = "0.16.1"
|
|
90
90
|
dependencies = [
|
|
91
91
|
"array-format",
|
|
92
92
|
"async-trait",
|
|
@@ -5,7 +5,7 @@ members = ["atlas-python"]
|
|
|
5
5
|
# Registry name on crates.io (`atlas` is taken). The library is still
|
|
6
6
|
# imported as `atlas` — see `[lib]` below.
|
|
7
7
|
name = "atlas-rust"
|
|
8
|
-
version = "0.16.
|
|
8
|
+
version = "0.16.1"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
|
|
11
11
|
license = "Apache-2.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: atlas-python
|
|
3
|
-
Version: 0.16.
|
|
3
|
+
Version: 0.16.1
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Science/Research
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -39,9 +39,9 @@ pip install atlas-python
|
|
|
39
39
|
```bash
|
|
40
40
|
atlas create /data/nc /data/collection
|
|
41
41
|
atlas ls /data/collection
|
|
42
|
-
atlas show /data/collection 2024-01
|
|
42
|
+
atlas show /data/collection 2024-01.nc
|
|
43
43
|
atlas info /data/collection
|
|
44
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
44
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
45
45
|
```
|
|
46
46
|
|
|
47
47
|
| Extra | Install | Adds |
|
|
@@ -58,10 +58,10 @@ The same five as a library:
|
|
|
58
58
|
import atlas
|
|
59
59
|
|
|
60
60
|
atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
|
|
61
|
-
atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
|
|
62
|
-
atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
|
|
61
|
+
atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
62
|
+
atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
|
|
63
63
|
atlas.info("/data/collection") # counts, size, codec, statistics
|
|
64
|
-
atlas.remove("/data/collection", ["2024-02"]) # updates the mask
|
|
64
|
+
atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
|
|
65
65
|
```
|
|
66
66
|
|
|
67
67
|
Every one takes a local path, a URL, or an obstore handle:
|
|
@@ -94,8 +94,8 @@ request.
|
|
|
94
94
|
## What `show` gives you
|
|
95
95
|
|
|
96
96
|
```text
|
|
97
|
-
$ atlas show /data/collection 2024-01
|
|
98
|
-
dataset 2024-01 {
|
|
97
|
+
$ atlas show /data/collection 2024-01.nc
|
|
98
|
+
dataset 2024-01.nc {
|
|
99
99
|
dimensions:
|
|
100
100
|
lat = 4 ;
|
|
101
101
|
lon = 6 ;
|
|
@@ -121,7 +121,7 @@ command gives the same content as a structure.
|
|
|
121
121
|
## Ingest
|
|
122
122
|
|
|
123
123
|
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
|
|
124
|
-
them, and writes one dataset per file, named after the
|
|
124
|
+
them, and writes one dataset per file, named after the file. Each coordinate
|
|
125
125
|
and data variable becomes an array. Each variable attribute becomes a per-array
|
|
126
126
|
attribute. `_FillValue` becomes the fill of the array.
|
|
127
127
|
|
|
@@ -134,6 +134,13 @@ Nothing at the destination is readable until every file lands. A failure
|
|
|
134
134
|
part-way leaves no collection, and not a partial one. `on_error="skip"`, or
|
|
135
135
|
`--skip-errors`, trades that for progress.
|
|
136
136
|
|
|
137
|
+
Atlas cannot store every numpy dtype, and `bool` is the common case. One such
|
|
138
|
+
variable fails the whole file by default. `on_unsupported="skip"`, or
|
|
139
|
+
`--skip-unsupported`, leaves out that one array and lands the rest.
|
|
140
|
+
|
|
141
|
+
`--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
|
|
142
|
+
warning to a file, each with its reason and the file it came from.
|
|
143
|
+
|
|
137
144
|
## dtypes
|
|
138
145
|
|
|
139
146
|
| numpy | atlas |
|
|
@@ -11,9 +11,9 @@ pip install atlas-python
|
|
|
11
11
|
```bash
|
|
12
12
|
atlas create /data/nc /data/collection
|
|
13
13
|
atlas ls /data/collection
|
|
14
|
-
atlas show /data/collection 2024-01
|
|
14
|
+
atlas show /data/collection 2024-01.nc
|
|
15
15
|
atlas info /data/collection
|
|
16
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
16
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
| Extra | Install | Adds |
|
|
@@ -30,10 +30,10 @@ The same five as a library:
|
|
|
30
30
|
import atlas
|
|
31
31
|
|
|
32
32
|
atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
|
|
33
|
-
atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
|
|
34
|
-
atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
|
|
33
|
+
atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
34
|
+
atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
|
|
35
35
|
atlas.info("/data/collection") # counts, size, codec, statistics
|
|
36
|
-
atlas.remove("/data/collection", ["2024-02"]) # updates the mask
|
|
36
|
+
atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
|
|
37
37
|
```
|
|
38
38
|
|
|
39
39
|
Every one takes a local path, a URL, or an obstore handle:
|
|
@@ -66,8 +66,8 @@ request.
|
|
|
66
66
|
## What `show` gives you
|
|
67
67
|
|
|
68
68
|
```text
|
|
69
|
-
$ atlas show /data/collection 2024-01
|
|
70
|
-
dataset 2024-01 {
|
|
69
|
+
$ atlas show /data/collection 2024-01.nc
|
|
70
|
+
dataset 2024-01.nc {
|
|
71
71
|
dimensions:
|
|
72
72
|
lat = 4 ;
|
|
73
73
|
lon = 6 ;
|
|
@@ -93,7 +93,7 @@ command gives the same content as a structure.
|
|
|
93
93
|
## Ingest
|
|
94
94
|
|
|
95
95
|
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
|
|
96
|
-
them, and writes one dataset per file, named after the
|
|
96
|
+
them, and writes one dataset per file, named after the file. Each coordinate
|
|
97
97
|
and data variable becomes an array. Each variable attribute becomes a per-array
|
|
98
98
|
attribute. `_FillValue` becomes the fill of the array.
|
|
99
99
|
|
|
@@ -106,6 +106,13 @@ Nothing at the destination is readable until every file lands. A failure
|
|
|
106
106
|
part-way leaves no collection, and not a partial one. `on_error="skip"`, or
|
|
107
107
|
`--skip-errors`, trades that for progress.
|
|
108
108
|
|
|
109
|
+
Atlas cannot store every numpy dtype, and `bool` is the common case. One such
|
|
110
|
+
variable fails the whole file by default. `on_unsupported="skip"`, or
|
|
111
|
+
`--skip-unsupported`, leaves out that one array and lands the rest.
|
|
112
|
+
|
|
113
|
+
`--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
|
|
114
|
+
warning to a file, each with its reason and the file it came from.
|
|
115
|
+
|
|
109
116
|
## dtypes
|
|
110
117
|
|
|
111
118
|
| numpy | atlas |
|
|
@@ -16,8 +16,27 @@ A `<collection>` is a local path or a URL: `s3://bucket/prefix`, `gs://...`,
|
|
|
16
16
|
`pip install "atlas-python[cloud]"`. See
|
|
17
17
|
[Cloud storage](guides/cloud-storage.md).
|
|
18
18
|
|
|
19
|
-
Every subcommand takes `--json
|
|
20
|
-
`--endpoint`, and `--anonymous`.
|
|
19
|
+
Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
|
|
20
|
+
`--region`, `--endpoint`, and `--anonymous`.
|
|
21
|
+
|
|
22
|
+
## Logging
|
|
23
|
+
|
|
24
|
+
`--log-file PATH` appends every error and warning to a file, with the reason:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
$ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
|
|
28
|
+
$ cat ingest.log
|
|
29
|
+
2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.1: create /data/nc ...
|
|
30
|
+
2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
|
|
31
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
|
|
32
|
+
2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The file opens in append mode. Each line names the file it came from, so a
|
|
36
|
+
large ingest stays readable.
|
|
37
|
+
|
|
38
|
+
This is separate from `ATLAS_LOG`, which turns on the Rust `tracing` stream to
|
|
39
|
+
stderr.
|
|
21
40
|
|
|
22
41
|
## create
|
|
23
42
|
|
|
@@ -25,9 +44,9 @@ Every subcommand takes `--json`, and the remote flags `--region`,
|
|
|
25
44
|
atlas create /data/nc /data/collection
|
|
26
45
|
```
|
|
27
46
|
|
|
28
|
-
Each NetCDF file becomes one dataset, named after
|
|
29
|
-
becomes `2024-01
|
|
30
|
-
a collection reproducible.
|
|
47
|
+
Each NetCDF file becomes one dataset, named after the file. `2024-01.nc`
|
|
48
|
+
becomes `2024-01.nc`, suffix and all. The files land in sorted order, which
|
|
49
|
+
makes the ordinals of a collection reproducible.
|
|
31
50
|
|
|
32
51
|
Nothing at the destination is readable until every file lands, with the footer.
|
|
33
52
|
A failure part-way leaves no collection, and not a partial one.
|
|
@@ -40,10 +59,33 @@ A failure part-way leaves no collection, and not a partial one.
|
|
|
40
59
|
| `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
|
|
41
60
|
| `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
|
|
42
61
|
| `--skip-errors` | Skip files that fail instead of abandoning the collection |
|
|
62
|
+
| `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
|
|
43
63
|
| `-q`, `--quiet` | Do not list a file as it lands |
|
|
44
64
|
|
|
45
65
|
Progress goes to stderr, so a pipe still reads stdout.
|
|
46
66
|
|
|
67
|
+
### Unsupported dtypes
|
|
68
|
+
|
|
69
|
+
Atlas cannot store every numpy dtype. `bool` is the common case. By default one
|
|
70
|
+
such variable fails the whole file:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
$ atlas create /data/nc /data/collection
|
|
74
|
+
atlas: /data/nc/buoy.nc: numpy dtype dtype('bool') is not supported by atlas ...
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`--skip-unsupported` narrows that to the one array:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
$ atlas create /data/nc /data/collection --skip-unsupported
|
|
81
|
+
skipped array buoy/flag (bool)
|
|
82
|
+
1 dataset(s) written to /data/collection
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Every other array of that dataset lands, with its attributes. `--json` reports
|
|
86
|
+
the skipped arrays under `skipped_arrays`, with the dataset, the dtype, and the
|
|
87
|
+
reason. See [Supported dtypes](guides/dtypes.md).
|
|
88
|
+
|
|
47
89
|
### Large files
|
|
48
90
|
|
|
49
91
|
Each file reads in dask blocks. A file far larger than memory therefore
|
|
@@ -88,7 +130,7 @@ atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
|
|
|
88
130
|
## rm
|
|
89
131
|
|
|
90
132
|
```bash
|
|
91
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
133
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
92
134
|
```
|
|
93
135
|
|
|
94
136
|
This removes several datasets in one call. A name is a dataset name, or the
|
|
@@ -114,9 +156,9 @@ error.
|
|
|
114
156
|
|
|
115
157
|
```bash
|
|
116
158
|
$ atlas ls /data/collection
|
|
117
|
-
2024-01
|
|
118
|
-
2024-02
|
|
119
|
-
2024-03
|
|
159
|
+
2024-01.nc
|
|
160
|
+
2024-02.nc
|
|
161
|
+
2024-03.nc
|
|
120
162
|
```
|
|
121
163
|
|
|
122
164
|
One name per line, in write order. A removed dataset does not appear. This
|
|
@@ -131,8 +173,8 @@ atlas ls /data/collection --json | jq . # as a JSON array
|
|
|
131
173
|
## show
|
|
132
174
|
|
|
133
175
|
```bash
|
|
134
|
-
$ atlas show /data/collection 2024-01
|
|
135
|
-
dataset 2024-01 {
|
|
176
|
+
$ atlas show /data/collection 2024-01.nc
|
|
177
|
+
dataset 2024-01.nc {
|
|
136
178
|
dimensions:
|
|
137
179
|
lat = 4 ;
|
|
138
180
|
lon = 6 ;
|
|
@@ -19,8 +19,8 @@ python atlas-python/examples/01_library.py
|
|
|
19
19
|
atlas create /data/nc /data/collection
|
|
20
20
|
atlas ls /data/collection
|
|
21
21
|
atlas info /data/collection
|
|
22
|
-
atlas show /data/collection 2024-01
|
|
23
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
22
|
+
atlas show /data/collection 2024-01.nc
|
|
23
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
See [The `atlas` command](cli.md).
|
|
@@ -51,8 +51,8 @@ store = obs.store.S3Store(
|
|
|
51
51
|
|
|
52
52
|
atlas.create("/data/nc", store)
|
|
53
53
|
atlas.list_datasets(store)
|
|
54
|
-
atlas.describe(store, "2024-01")
|
|
55
|
-
atlas.remove(store, ["2024-02"])
|
|
54
|
+
atlas.describe(store, "2024-01.nc")
|
|
55
|
+
atlas.remove(store, ["2024-02.nc"])
|
|
56
56
|
atlas.info(store)
|
|
57
57
|
```
|
|
58
58
|
|
|
@@ -15,8 +15,9 @@ atlas create /data/nc /data/collection
|
|
|
15
15
|
## What happens
|
|
16
16
|
|
|
17
17
|
`create` collects every file that matches `.nc`, `.nc4`, `.cdf`, or `.netcdf`.
|
|
18
|
-
It **sorts** them, and writes one dataset per file, named after the file
|
|
19
|
-
`2024-01.nc` becomes the dataset `2024-01`.
|
|
18
|
+
It **sorts** them, and writes one dataset per file, named after the file.
|
|
19
|
+
`2024-01.nc` becomes the dataset `2024-01.nc`. The suffix is part of the name,
|
|
20
|
+
so `jan.nc` and `jan.nc4` are two datasets.
|
|
20
21
|
|
|
21
22
|
The sort matters. An ordinal comes from the write order, so a sorted ingest
|
|
22
23
|
makes it reproducible. Rebuild the same directory, and every dataset lands at
|
|
@@ -39,7 +40,7 @@ That is what you usually want. When it is not:
|
|
|
39
40
|
|
|
40
41
|
```python
|
|
41
42
|
result = atlas.create("/data/nc", dest, on_error="skip")
|
|
42
|
-
result["written"] # ['2024-01', '2024-03']
|
|
43
|
+
result["written"] # ['2024-01.nc', '2024-03.nc']
|
|
43
44
|
result["skipped"] # [{'file': '.../2024-02.nc', 'error': '...'}]
|
|
44
45
|
```
|
|
45
46
|
|
|
@@ -51,6 +52,78 @@ A skipped file leaves no trace in the collection. The writer moves to the next
|
|
|
51
52
|
one. The CLI exits `1` when it skipped anything, so a pipeline sees that. It
|
|
52
53
|
still writes the collection.
|
|
53
54
|
|
|
55
|
+
## One bad array, not one bad file
|
|
56
|
+
|
|
57
|
+
`on_error` works at the granularity of a file. One variable of an unsupported
|
|
58
|
+
dtype therefore costs the whole dataset. `on_unsupported="skip"` narrows that
|
|
59
|
+
to the array:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
result = atlas.create("/data/nc", dest, on_unsupported="skip")
|
|
63
|
+
result["skipped_arrays"]
|
|
64
|
+
# [{'array': 'flag', 'dtype': 'bool', 'error': '...', 'dataset': '2024-01.nc'}]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
atlas create /data/nc /data/collection --skip-unsupported
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The rest of the dataset lands as usual: every other array, every attribute,
|
|
72
|
+
and the dataset itself. The skipped name is absent from the schema, so no
|
|
73
|
+
empty array stands in for it.
|
|
74
|
+
|
|
75
|
+
Atlas resolves every dtype before it defines the first array. A skip therefore
|
|
76
|
+
never leaves a half-written array behind.
|
|
77
|
+
|
|
78
|
+
The two settings compose. `--skip-unsupported` handles the array atlas cannot
|
|
79
|
+
store. `--skip-errors` handles the file that fails for any other reason.
|
|
80
|
+
|
|
81
|
+
See [Supported dtypes](dtypes.md) for what atlas can store.
|
|
82
|
+
|
|
83
|
+
## The log file
|
|
84
|
+
|
|
85
|
+
Both kinds of skip go to a log file, with the reason:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
```text
|
|
92
|
+
2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
|
|
93
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/broken.nc: ValueError: did not find a match ...
|
|
94
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
|
|
95
|
+
2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 1 file(s) and 1 array(s)
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Each line names the file, so a thousand-file ingest stays readable. The file
|
|
99
|
+
opens in append mode, so a repeat run adds to it.
|
|
100
|
+
|
|
101
|
+
From the library:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
import atlas
|
|
105
|
+
|
|
106
|
+
atlas.log_to_file("ingest.log")
|
|
107
|
+
atlas.create("/data/nc", dest, on_unsupported="skip")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Atlas logs to the `atlas` logger and attaches no handler of its own, so attach
|
|
111
|
+
your own to send the records somewhere else:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
import logging
|
|
115
|
+
|
|
116
|
+
logging.getLogger("atlas").addHandler(logging.StreamHandler())
|
|
117
|
+
logging.getLogger("atlas").setLevel(logging.INFO)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`log_to_file` also captures Python warnings, such as the one about missing
|
|
121
|
+
string cells. That moves them off stderr, because `logging.captureWarnings` is
|
|
122
|
+
process-wide.
|
|
123
|
+
|
|
124
|
+
The Rust core logs separately, through `tracing`. `atlas.init_tracing()` sends
|
|
125
|
+
that stream to stderr. See [Installation](../installation.md).
|
|
126
|
+
|
|
54
127
|
## Chunking and memory
|
|
55
128
|
|
|
56
129
|
These are one decision. The blocks a file *reads* in are the chunks it
|
|
@@ -114,12 +187,12 @@ alone. A one-chunk array reads whole, or not at all.
|
|
|
114
187
|
Confirm what landed:
|
|
115
188
|
|
|
116
189
|
```python
|
|
117
|
-
arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01")["arrays"]}
|
|
190
|
+
arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01.nc")["arrays"]}
|
|
118
191
|
arrays["temperature"]["chunk_shape"]
|
|
119
192
|
```
|
|
120
193
|
|
|
121
194
|
```bash
|
|
122
|
-
atlas show dest 2024-01 | grep _ChunkShape
|
|
195
|
+
atlas show dest 2024-01.nc | grep _ChunkShape
|
|
123
196
|
```
|
|
124
197
|
|
|
125
198
|
### The writer's own memory
|
|
@@ -157,12 +230,13 @@ turns it off.
|
|
|
157
230
|
| Situation | Result |
|
|
158
231
|
|---|---|
|
|
159
232
|
| Directory holds no NetCDF files | `AtlasError` |
|
|
160
|
-
| Two files share a
|
|
161
|
-
| A variable has a dtype atlas cannot store | `AtlasError
|
|
233
|
+
| Two files share a name | `AtlasError`. A dataset name must be unique |
|
|
234
|
+
| A variable has a dtype atlas cannot store | `AtlasError`, or one skipped array under `--skip-unsupported` |
|
|
235
|
+
| Any other bad file | `AtlasError`, or one skipped file under `--skip-errors` |
|
|
162
236
|
| Destination URL cannot be resolved | `SourceError` |
|
|
163
237
|
|
|
164
|
-
Two files with one
|
|
165
|
-
the name `jan`. Rename one, or ingest them into two collections.
|
|
238
|
+
Two files with one name surprise people. `a/jan.nc` and `b/jan.nc` both want
|
|
239
|
+
the name `jan.nc`. Rename one, or ingest them into two collections.
|
|
166
240
|
|
|
167
241
|
For the dtype rules, see [Supported dtypes](dtypes.md).
|
|
168
242
|
|
|
@@ -62,7 +62,7 @@ fill of the array, and not as an attribute.
|
|
|
62
62
|
Check what landed:
|
|
63
63
|
|
|
64
64
|
```python
|
|
65
|
-
arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01")["arrays"]}
|
|
65
|
+
arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01.nc")["arrays"]}
|
|
66
66
|
arrays["temperature"]["fill_value"]
|
|
67
67
|
```
|
|
68
68
|
|
|
@@ -86,7 +86,10 @@ array data, not to the attributes. They come through unchanged:
|
|
|
86
86
|
| `list[...]`, `fixed_size_list[..., N]` | Not exposed yet |
|
|
87
87
|
|
|
88
88
|
A NetCDF file with a boolean variable fails the ingest. Store it as `uint8`,
|
|
89
|
-
and write the convention down.
|
|
89
|
+
and write the convention down. To keep the rest of that file, pass
|
|
90
|
+
`--skip-unsupported`, or `on_unsupported="skip"`. That leaves out the one array
|
|
91
|
+
and lands everything else. See
|
|
92
|
+
[Creating a collection](creating.md#one-bad-array-not-one-bad-file).
|
|
90
93
|
|
|
91
94
|
All four work as an **attribute** value, which is where they usually appear.
|
|
92
95
|
|
|
@@ -100,7 +103,7 @@ Some xarray attributes are no scalar. A nested dict, a ragged list, and a numpy
|
|
|
100
103
|
array each encode as JSON on the way in. `describe` decodes them again:
|
|
101
104
|
|
|
102
105
|
```python
|
|
103
|
-
atlas.describe(collection, "2024-01")["attributes"]
|
|
106
|
+
atlas.describe(collection, "2024-01.nc")["attributes"]
|
|
104
107
|
# {'month': 1, 'bounds': [1.0, 2.0], 'nested': {'a': 1}}
|
|
105
108
|
```
|
|
106
109
|
|
|
@@ -7,7 +7,7 @@ fetches array data, so a catalogue of ten thousand datasets costs one request.
|
|
|
7
7
|
|
|
8
8
|
```python
|
|
9
9
|
atlas.list_datasets("/data/collection")
|
|
10
|
-
# ['2024-01', '2024-02', '2024-03']
|
|
10
|
+
# ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
```bash
|
|
@@ -63,11 +63,11 @@ Use `describe` for the statistics of one dataset on its own.
|
|
|
63
63
|
## One dataset in detail: `describe` and `show`
|
|
64
64
|
|
|
65
65
|
```python
|
|
66
|
-
atlas.describe("/data/collection", "2024-01")
|
|
66
|
+
atlas.describe("/data/collection", "2024-01.nc")
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
atlas show /data/collection 2024-01
|
|
70
|
+
atlas show /data/collection 2024-01.nc
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
The CLI prints it like `ncdump -h`. The library returns the structure. Both
|
|
@@ -78,7 +78,7 @@ attributes, the coordinate flag, and the statistics.
|
|
|
78
78
|
`name` is a dataset name, or the NetCDF path the dataset came from:
|
|
79
79
|
|
|
80
80
|
```python
|
|
81
|
-
atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01"
|
|
81
|
+
atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01.nc"
|
|
82
82
|
```
|
|
83
83
|
|
|
84
84
|
### Statistics
|
|
@@ -117,7 +117,7 @@ The ingest records which variables were xarray coordinates, and reports them
|
|
|
117
117
|
back:
|
|
118
118
|
|
|
119
119
|
```python
|
|
120
|
-
detail = atlas.describe(collection, "2024-01")
|
|
120
|
+
detail = atlas.describe(collection, "2024-01.nc")
|
|
121
121
|
detail["coordinates"] # ['lat', 'lon']
|
|
122
122
|
[a["name"] for a in detail["arrays"] if a["is_coordinate"]]
|
|
123
123
|
```
|
|
@@ -4,10 +4,10 @@ Python reads the **metadata** of a collection. It reads no array value.
|
|
|
4
4
|
|
|
5
5
|
```python
|
|
6
6
|
atlas.list_datasets(collection) # ✓
|
|
7
|
-
atlas.describe(collection, "jan")
|
|
7
|
+
atlas.describe(collection, "jan.nc") # ✓ types, shapes, attrs, stats
|
|
8
8
|
atlas.info(collection) # ✓ counts, size, collection stats
|
|
9
9
|
|
|
10
|
-
atlas.read_array(collection, "jan", "t")
|
|
10
|
+
atlas.read_array(collection, "jan.nc", "t") # ✗ does not exist
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
## Why
|
|
@@ -27,7 +27,7 @@ It is enough to build a catalogue, to check an ingest, or to choose the
|
|
|
27
27
|
datasets worth a fetch:
|
|
28
28
|
|
|
29
29
|
```python
|
|
30
|
-
detail = atlas.describe(collection, "2024-01")
|
|
30
|
+
detail = atlas.describe(collection, "2024-01.nc")
|
|
31
31
|
|
|
32
32
|
detail["dimensions"] # {'lat': 4, 'lon': 6}
|
|
33
33
|
detail["coordinates"] # ['lat', 'lon']
|
|
@@ -99,7 +99,7 @@ simpler than anything else here.
|
|
|
99
99
|
reader opens it:
|
|
100
100
|
|
|
101
101
|
```python
|
|
102
|
-
start, end = atlas.describe(collection, "2024-01")["segment_range"]
|
|
102
|
+
start, end = atlas.describe(collection, "2024-01.nc")["segment_range"]
|
|
103
103
|
blob = open(f"{collection}/data.atlas", "rb").read()[start:end]
|
|
104
104
|
open("2024-01.af", "wb").write(blob)
|
|
105
105
|
```
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# Removing datasets
|
|
2
2
|
|
|
3
3
|
```python
|
|
4
|
-
atlas.remove("/data/collection", ["2024-02", "2024-03"])
|
|
4
|
+
atlas.remove("/data/collection", ["2024-02.nc", "2024-03.nc"])
|
|
5
5
|
```
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
8
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
One call, whatever the number of datasets. A name is a dataset name, or the
|
|
@@ -66,7 +66,7 @@ mask holds a sorted set of ordinals.
|
|
|
66
66
|
The same holds on the command line, up to the argument limit of your shell:
|
|
67
67
|
|
|
68
68
|
```bash
|
|
69
|
-
atlas rm /data/collection 2024-01 2024-02 2024-03
|
|
69
|
+
atlas rm /data/collection 2024-01.nc 2024-02.nc 2024-03.nc
|
|
70
70
|
```
|
|
71
71
|
|
|
72
72
|
For a list too long for one command line, call `atlas.remove` from Python.
|
|
@@ -83,8 +83,8 @@ atlas.remove(collection, ["nope"])
|
|
|
83
83
|
`missing_ok` reports it instead:
|
|
84
84
|
|
|
85
85
|
```python
|
|
86
|
-
result = atlas.remove(collection, ["2024-01", "nope"], missing_ok=True)
|
|
87
|
-
result["removed"] # ['2024-01']
|
|
86
|
+
result = atlas.remove(collection, ["2024-01.nc", "nope"], missing_ok=True)
|
|
87
|
+
result["removed"] # ['2024-01.nc']
|
|
88
88
|
result["missing"] # ['nope']
|
|
89
89
|
```
|
|
90
90
|
|
|
@@ -10,9 +10,9 @@ atlas create /data/nc /data/collection
|
|
|
10
10
|
|
|
11
11
|
```text
|
|
12
12
|
$ atlas ls /data/collection
|
|
13
|
-
2024-01
|
|
14
|
-
2024-02
|
|
15
|
-
2024-03
|
|
13
|
+
2024-01.nc
|
|
14
|
+
2024-02.nc
|
|
15
|
+
2024-03.nc
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
## The shape of it
|
|
@@ -60,8 +60,8 @@ atlas.list_datasets("s3://bucket/2024", region="eu-west-1")
|
|
|
60
60
|
## What `show` gives you
|
|
61
61
|
|
|
62
62
|
```text
|
|
63
|
-
$ atlas show /data/collection 2024-01
|
|
64
|
-
dataset 2024-01 {
|
|
63
|
+
$ atlas show /data/collection 2024-01.nc
|
|
64
|
+
dataset 2024-01.nc {
|
|
65
65
|
dimensions:
|
|
66
66
|
lat = 4 ;
|
|
67
67
|
lon = 6 ;
|
|
@@ -10,13 +10,13 @@ atlas create /data/nc /data/collection
|
|
|
10
10
|
|
|
11
11
|
```text
|
|
12
12
|
Writing /data/collection
|
|
13
|
-
2024-01
|
|
14
|
-
2024-02
|
|
15
|
-
2024-03
|
|
13
|
+
2024-01.nc
|
|
14
|
+
2024-02.nc
|
|
15
|
+
2024-03.nc
|
|
16
16
|
3 dataset(s) written to /data/collection
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
Each file becomes one dataset, named after
|
|
19
|
+
Each file becomes one dataset, named after the file. The result is one file:
|
|
20
20
|
|
|
21
21
|
```bash
|
|
22
22
|
$ ls /data/collection
|
|
@@ -38,9 +38,9 @@ atlas.create("/data/nc", "/data/collection")
|
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
40
|
$ atlas ls /data/collection
|
|
41
|
-
2024-01
|
|
42
|
-
2024-02
|
|
43
|
-
2024-03
|
|
41
|
+
2024-01.nc
|
|
42
|
+
2024-02.nc
|
|
43
|
+
2024-03.nc
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
```bash
|
|
@@ -72,8 +72,8 @@ The maximum comes from March. Removed datasets do not count.
|
|
|
72
72
|
## Inspect one dataset
|
|
73
73
|
|
|
74
74
|
```bash
|
|
75
|
-
$ atlas show /data/collection 2024-01
|
|
76
|
-
dataset 2024-01 {
|
|
75
|
+
$ atlas show /data/collection 2024-01.nc
|
|
76
|
+
dataset 2024-01.nc {
|
|
77
77
|
dimensions:
|
|
78
78
|
lat = 4 ;
|
|
79
79
|
lon = 6 ;
|
|
@@ -103,7 +103,7 @@ footer holds those too, so to print them costs nothing extra.
|
|
|
103
103
|
From Python, the same thing as a structure:
|
|
104
104
|
|
|
105
105
|
```python
|
|
106
|
-
detail = atlas.describe("/data/collection", "2024-01")
|
|
106
|
+
detail = atlas.describe("/data/collection", "2024-01.nc")
|
|
107
107
|
detail["dimensions"] # {'lat': 4, 'lon': 6}
|
|
108
108
|
detail["coordinates"] # ['lat', 'lon']
|
|
109
109
|
{a["name"]: a["stats"] for a in detail["arrays"]}
|
|
@@ -112,8 +112,8 @@ detail["coordinates"] # ['lat', 'lon']
|
|
|
112
112
|
## Remove
|
|
113
113
|
|
|
114
114
|
```bash
|
|
115
|
-
$ atlas rm /data/collection 2024-02
|
|
116
|
-
removed 1: 2024-02
|
|
115
|
+
$ atlas rm /data/collection 2024-02.nc
|
|
116
|
+
removed 1: 2024-02.nc
|
|
117
117
|
2 dataset(s) remain
|
|
118
118
|
```
|
|
119
119
|
|
|
@@ -139,7 +139,7 @@ Not from Python. The Rust API reads array values:
|
|
|
139
139
|
|
|
140
140
|
```rust
|
|
141
141
|
let atlas = Atlas::open_path("/data/collection").await?;
|
|
142
|
-
let ds = atlas.dataset("2024-01")?;
|
|
142
|
+
let ds = atlas.dataset("2024-01.nc")?;
|
|
143
143
|
let window = ds.read_array::<f32>("temperature", vec![0, 0], vec![2, 3]).await?;
|
|
144
144
|
```
|
|
145
145
|
|