atlas-python 0.16.0__tar.gz → 0.16.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {atlas_python-0.16.0 → atlas_python-0.16.2}/Cargo.lock +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.2}/Cargo.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.2}/PKG-INFO +21 -10
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/Cargo.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/README.md +20 -9
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/cli.md +62 -13
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/examples.md +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/cloud-storage.md +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/creating.md +102 -11
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/dtypes.md +6 -3
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/inspecting.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/reading-data.md +4 -4
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/removing.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/index.md +5 -5
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/installation.md +41 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/quickstart.md +13 -13
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/reference/api.md +4 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/vs-zarr-netcdf.md +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/examples/01_library.py +4 -4
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/examples/02_object_store.py +2 -2
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/conftest.py +20 -1
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_cli.py +126 -17
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_ops.py +209 -62
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_source.py +3 -3
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/format.md +10 -3
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/python.md +35 -5
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/write-path.md +1 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/pyproject.toml +1 -1
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/__init__.py +2 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/__init__.pyi +54 -12
- atlas_python-0.16.2/python/atlas/__main__.py +14 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_cli.py +54 -10
- atlas_python-0.16.2/python/atlas/_log.py +82 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_ops.py +73 -17
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/xarray.py +64 -11
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/footer.rs +39 -3
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/writer/mod.rs +10 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/cross_fixture.rs +6 -6
- {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/integration.rs +25 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-python-docs.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-python-release.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-rust-release.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/ci.yaml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/.gitignore +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/CONTRIBUTING.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/LICENSE +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/README.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/.python-version +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/LICENSE +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/mkdocs.yml +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/attr.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/dtype.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/error.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/lib.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/logging.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/reader.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/runtime.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/source.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/writer.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/make_fixture.py +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/README.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/architecture.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/data-model.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/read-path.md +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/lifecycle.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/sensor_fleet.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/weather_store.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_source.py +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/py.typed +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/config.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/error.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/mask.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/segment_store.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/lib.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/reader/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/array.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/attr.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/dtype.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/mod.rs +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/fixtures/from_python/data.atlas +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/fixtures/golden_v1/data.atlas +0 -0
- {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/golden.rs +0 -0
|
@@ -71,7 +71,7 @@ dependencies = [
|
|
|
71
71
|
|
|
72
72
|
[[package]]
|
|
73
73
|
name = "atlas-python"
|
|
74
|
-
version = "0.16.
|
|
74
|
+
version = "0.16.2"
|
|
75
75
|
dependencies = [
|
|
76
76
|
"atlas-rust",
|
|
77
77
|
"ndarray",
|
|
@@ -86,7 +86,7 @@ dependencies = [
|
|
|
86
86
|
|
|
87
87
|
[[package]]
|
|
88
88
|
name = "atlas-rust"
|
|
89
|
-
version = "0.16.
|
|
89
|
+
version = "0.16.2"
|
|
90
90
|
dependencies = [
|
|
91
91
|
"array-format",
|
|
92
92
|
"async-trait",
|
|
@@ -5,7 +5,7 @@ members = ["atlas-python"]
|
|
|
5
5
|
# Registry name on crates.io (`atlas` is taken). The library is still
|
|
6
6
|
# imported as `atlas` — see `[lib]` below.
|
|
7
7
|
name = "atlas-rust"
|
|
8
|
-
version = "0.16.
|
|
8
|
+
version = "0.16.2"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
|
|
11
11
|
license = "Apache-2.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: atlas-python
|
|
3
|
-
Version: 0.16.
|
|
3
|
+
Version: 0.16.2
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Intended Audience :: Science/Research
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -39,11 +39,14 @@ pip install atlas-python
|
|
|
39
39
|
```bash
|
|
40
40
|
atlas create /data/nc /data/collection
|
|
41
41
|
atlas ls /data/collection
|
|
42
|
-
atlas show /data/collection 2024-01
|
|
42
|
+
atlas show /data/collection 2024-01.nc
|
|
43
43
|
atlas info /data/collection
|
|
44
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
44
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
45
45
|
```
|
|
46
46
|
|
|
47
|
+
`python -m atlas` runs the same command without a PATH lookup, for a shell that
|
|
48
|
+
cannot find `atlas`.
|
|
49
|
+
|
|
47
50
|
| Extra | Install | Adds |
|
|
48
51
|
|---|---|---|
|
|
49
52
|
| cloud | `pip install "atlas-python[cloud]"` | S3 / GCS / Azure / HTTP via [obstore](https://github.com/developmentseed/obstore) |
|
|
@@ -58,10 +61,10 @@ The same five as a library:
|
|
|
58
61
|
import atlas
|
|
59
62
|
|
|
60
63
|
atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
|
|
61
|
-
atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
|
|
62
|
-
atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
|
|
64
|
+
atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
65
|
+
atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
|
|
63
66
|
atlas.info("/data/collection") # counts, size, codec, statistics
|
|
64
|
-
atlas.remove("/data/collection", ["2024-02"]) # updates the mask
|
|
67
|
+
atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
|
|
65
68
|
```
|
|
66
69
|
|
|
67
70
|
Every one takes a local path, a URL, or an obstore handle:
|
|
@@ -94,8 +97,8 @@ request.
|
|
|
94
97
|
## What `show` gives you
|
|
95
98
|
|
|
96
99
|
```text
|
|
97
|
-
$ atlas show /data/collection 2024-01
|
|
98
|
-
dataset 2024-01 {
|
|
100
|
+
$ atlas show /data/collection 2024-01.nc
|
|
101
|
+
dataset 2024-01.nc {
|
|
99
102
|
dimensions:
|
|
100
103
|
lat = 4 ;
|
|
101
104
|
lon = 6 ;
|
|
@@ -120,8 +123,9 @@ command gives the same content as a structure.
|
|
|
120
123
|
|
|
121
124
|
## Ingest
|
|
122
125
|
|
|
123
|
-
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf
|
|
124
|
-
them, and writes one dataset per
|
|
126
|
+
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`, and
|
|
127
|
+
descends into every subdirectory. It sorts them, and writes one dataset per
|
|
128
|
+
file, named after the file. Each coordinate
|
|
125
129
|
and data variable becomes an array. Each variable attribute becomes a per-array
|
|
126
130
|
attribute. `_FillValue` becomes the fill of the array.
|
|
127
131
|
|
|
@@ -134,6 +138,13 @@ Nothing at the destination is readable until every file lands. A failure
|
|
|
134
138
|
part-way leaves no collection, and not a partial one. `on_error="skip"`, or
|
|
135
139
|
`--skip-errors`, trades that for progress.
|
|
136
140
|
|
|
141
|
+
Atlas cannot store every numpy dtype, and `bool` is the common case. One such
|
|
142
|
+
variable fails the whole file by default. `on_unsupported="skip"`, or
|
|
143
|
+
`--skip-unsupported`, leaves out that one array and lands the rest.
|
|
144
|
+
|
|
145
|
+
`--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
|
|
146
|
+
warning to a file, each with its reason and the file it came from.
|
|
147
|
+
|
|
137
148
|
## dtypes
|
|
138
149
|
|
|
139
150
|
| numpy | atlas |
|
|
@@ -11,11 +11,14 @@ pip install atlas-python
|
|
|
11
11
|
```bash
|
|
12
12
|
atlas create /data/nc /data/collection
|
|
13
13
|
atlas ls /data/collection
|
|
14
|
-
atlas show /data/collection 2024-01
|
|
14
|
+
atlas show /data/collection 2024-01.nc
|
|
15
15
|
atlas info /data/collection
|
|
16
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
16
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
+
`python -m atlas` runs the same command without a PATH lookup, for a shell that
|
|
20
|
+
cannot find `atlas`.
|
|
21
|
+
|
|
19
22
|
| Extra | Install | Adds |
|
|
20
23
|
|---|---|---|
|
|
21
24
|
| cloud | `pip install "atlas-python[cloud]"` | S3 / GCS / Azure / HTTP via [obstore](https://github.com/developmentseed/obstore) |
|
|
@@ -30,10 +33,10 @@ The same five as a library:
|
|
|
30
33
|
import atlas
|
|
31
34
|
|
|
32
35
|
atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
|
|
33
|
-
atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
|
|
34
|
-
atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
|
|
36
|
+
atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
37
|
+
atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
|
|
35
38
|
atlas.info("/data/collection") # counts, size, codec, statistics
|
|
36
|
-
atlas.remove("/data/collection", ["2024-02"]) # updates the mask
|
|
39
|
+
atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
|
|
37
40
|
```
|
|
38
41
|
|
|
39
42
|
Every one takes a local path, a URL, or an obstore handle:
|
|
@@ -66,8 +69,8 @@ request.
|
|
|
66
69
|
## What `show` gives you
|
|
67
70
|
|
|
68
71
|
```text
|
|
69
|
-
$ atlas show /data/collection 2024-01
|
|
70
|
-
dataset 2024-01 {
|
|
72
|
+
$ atlas show /data/collection 2024-01.nc
|
|
73
|
+
dataset 2024-01.nc {
|
|
71
74
|
dimensions:
|
|
72
75
|
lat = 4 ;
|
|
73
76
|
lon = 6 ;
|
|
@@ -92,8 +95,9 @@ command gives the same content as a structure.
|
|
|
92
95
|
|
|
93
96
|
## Ingest
|
|
94
97
|
|
|
95
|
-
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf
|
|
96
|
-
them, and writes one dataset per
|
|
98
|
+
`create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`, and
|
|
99
|
+
descends into every subdirectory. It sorts them, and writes one dataset per
|
|
100
|
+
file, named after the file. Each coordinate
|
|
97
101
|
and data variable becomes an array. Each variable attribute becomes a per-array
|
|
98
102
|
attribute. `_FillValue` becomes the fill of the array.
|
|
99
103
|
|
|
@@ -106,6 +110,13 @@ Nothing at the destination is readable until every file lands. A failure
|
|
|
106
110
|
part-way leaves no collection, and not a partial one. `on_error="skip"`, or
|
|
107
111
|
`--skip-errors`, trades that for progress.
|
|
108
112
|
|
|
113
|
+
Atlas cannot store every numpy dtype, and `bool` is the common case. One such
|
|
114
|
+
variable fails the whole file by default. `on_unsupported="skip"`, or
|
|
115
|
+
`--skip-unsupported`, leaves out that one array and lands the rest.
|
|
116
|
+
|
|
117
|
+
`--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
|
|
118
|
+
warning to a file, each with its reason and the file it came from.
|
|
119
|
+
|
|
109
120
|
## dtypes
|
|
110
121
|
|
|
111
122
|
| numpy | atlas |
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
`pip install atlas-python` puts `atlas` on your PATH. Five subcommands, one per
|
|
4
4
|
operation.
|
|
5
5
|
|
|
6
|
+
`python -m atlas` runs the same command without a PATH lookup. Use it in a
|
|
7
|
+
container, in a CI job, or when the shell cannot find `atlas`. See
|
|
8
|
+
[Installation](installation.md#the-atlas-command-is-not-found).
|
|
9
|
+
|
|
6
10
|
```text
|
|
7
11
|
atlas create <netcdf-dir> <collection> build a collection
|
|
8
12
|
atlas rm <collection> <name>... remove datasets
|
|
@@ -16,8 +20,27 @@ A `<collection>` is a local path or a URL: `s3://bucket/prefix`, `gs://...`,
|
|
|
16
20
|
`pip install "atlas-python[cloud]"`. See
|
|
17
21
|
[Cloud storage](guides/cloud-storage.md).
|
|
18
22
|
|
|
19
|
-
Every subcommand takes `--json
|
|
20
|
-
`--endpoint`, and `--anonymous`.
|
|
23
|
+
Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
|
|
24
|
+
`--region`, `--endpoint`, and `--anonymous`.
|
|
25
|
+
|
|
26
|
+
## Logging
|
|
27
|
+
|
|
28
|
+
`--log-file PATH` appends every error and warning to a file, with the reason:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
$ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
|
|
32
|
+
$ cat ingest.log
|
|
33
|
+
2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.2: create /data/nc ...
|
|
34
|
+
2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
|
|
35
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
|
|
36
|
+
2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The file opens in append mode. Each line names the file it came from, so a
|
|
40
|
+
large ingest stays readable.
|
|
41
|
+
|
|
42
|
+
This is separate from `ATLAS_LOG`, which turns on the Rust `tracing` stream to
|
|
43
|
+
stderr.
|
|
21
44
|
|
|
22
45
|
## create
|
|
23
46
|
|
|
@@ -25,25 +48,51 @@ Every subcommand takes `--json`, and the remote flags `--region`,
|
|
|
25
48
|
atlas create /data/nc /data/collection
|
|
26
49
|
```
|
|
27
50
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
51
|
+
The scan descends into every subdirectory. Each NetCDF file becomes one
|
|
52
|
+
dataset, named after the file. `2024-01.nc` becomes `2024-01.nc`, suffix and
|
|
53
|
+
all. A name carries no directory, so two files of one name in two
|
|
54
|
+
subdirectories collide. The files land in sorted order, which makes the
|
|
55
|
+
ordinals of a collection reproducible.
|
|
31
56
|
|
|
32
57
|
Nothing at the destination is readable until every file lands, with the footer.
|
|
33
58
|
A failure part-way leaves no collection, and not a partial one.
|
|
34
59
|
|
|
35
60
|
| Flag | Effect |
|
|
36
61
|
|---|---|
|
|
37
|
-
|
|
|
62
|
+
| `--no-recursive` | Scan the top directory alone. The scan descends by default |
|
|
63
|
+
| `-r`, `--recursive` | Accepted for compatibility. The scan already descends |
|
|
38
64
|
| `--codec {zstd,lz4,none}` | Block compression. Default `zstd` |
|
|
39
65
|
| `--chunk-size SIZE` | Block size to aim for. Default `128MiB` |
|
|
40
66
|
| `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
|
|
41
67
|
| `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
|
|
42
68
|
| `--skip-errors` | Skip files that fail instead of abandoning the collection |
|
|
69
|
+
| `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
|
|
43
70
|
| `-q`, `--quiet` | Do not list a file as it lands |
|
|
44
71
|
|
|
45
72
|
Progress goes to stderr, so a pipe still reads stdout.
|
|
46
73
|
|
|
74
|
+
### Unsupported dtypes
|
|
75
|
+
|
|
76
|
+
Atlas cannot store every numpy dtype. `bool` is the common case. By default one
|
|
77
|
+
such variable fails the whole file:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
$ atlas create /data/nc /data/collection
|
|
81
|
+
atlas: /data/nc/buoy.nc: numpy dtype dtype('bool') is not supported by atlas ...
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`--skip-unsupported` narrows that to the one array:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
$ atlas create /data/nc /data/collection --skip-unsupported
|
|
88
|
+
skipped array buoy/flag (bool)
|
|
89
|
+
1 dataset(s) written to /data/collection
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Every other array of that dataset lands, with its attributes. `--json` reports
|
|
93
|
+
the skipped arrays under `skipped_arrays`, with the dataset, the dtype, and the
|
|
94
|
+
reason. See [Supported dtypes](guides/dtypes.md).
|
|
95
|
+
|
|
47
96
|
### Large files
|
|
48
97
|
|
|
49
98
|
Each file reads in dask blocks. A file far larger than memory therefore
|
|
@@ -79,7 +128,7 @@ reads. Each misaligned block then costs a read-modify-write. Use
|
|
|
79
128
|
|
|
80
129
|
```bash
|
|
81
130
|
# One collection from a tree of monthly directories, tolerating bad files
|
|
82
|
-
atlas create /data/nc /data/collection --
|
|
131
|
+
atlas create /data/nc /data/collection --skip-errors
|
|
83
132
|
|
|
84
133
|
# A big grid, chunked for selective reads, straight to a bucket
|
|
85
134
|
atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
|
|
@@ -88,7 +137,7 @@ atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
|
|
|
88
137
|
## rm
|
|
89
138
|
|
|
90
139
|
```bash
|
|
91
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
140
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
92
141
|
```
|
|
93
142
|
|
|
94
143
|
This removes several datasets in one call. A name is a dataset name, or the
|
|
@@ -114,9 +163,9 @@ error.
|
|
|
114
163
|
|
|
115
164
|
```bash
|
|
116
165
|
$ atlas ls /data/collection
|
|
117
|
-
2024-01
|
|
118
|
-
2024-02
|
|
119
|
-
2024-03
|
|
166
|
+
2024-01.nc
|
|
167
|
+
2024-02.nc
|
|
168
|
+
2024-03.nc
|
|
120
169
|
```
|
|
121
170
|
|
|
122
171
|
One name per line, in write order. A removed dataset does not appear. This
|
|
@@ -131,8 +180,8 @@ atlas ls /data/collection --json | jq . # as a JSON array
|
|
|
131
180
|
## show
|
|
132
181
|
|
|
133
182
|
```bash
|
|
134
|
-
$ atlas show /data/collection 2024-01
|
|
135
|
-
dataset 2024-01 {
|
|
183
|
+
$ atlas show /data/collection 2024-01.nc
|
|
184
|
+
dataset 2024-01.nc {
|
|
136
185
|
dimensions:
|
|
137
186
|
lat = 4 ;
|
|
138
187
|
lon = 6 ;
|
|
@@ -19,8 +19,8 @@ python atlas-python/examples/01_library.py
|
|
|
19
19
|
atlas create /data/nc /data/collection
|
|
20
20
|
atlas ls /data/collection
|
|
21
21
|
atlas info /data/collection
|
|
22
|
-
atlas show /data/collection 2024-01
|
|
23
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
22
|
+
atlas show /data/collection 2024-01.nc
|
|
23
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
See [The `atlas` command](cli.md).
|
|
@@ -51,8 +51,8 @@ store = obs.store.S3Store(
|
|
|
51
51
|
|
|
52
52
|
atlas.create("/data/nc", store)
|
|
53
53
|
atlas.list_datasets(store)
|
|
54
|
-
atlas.describe(store, "2024-01")
|
|
55
|
-
atlas.remove(store, ["2024-02"])
|
|
54
|
+
atlas.describe(store, "2024-01.nc")
|
|
55
|
+
atlas.remove(store, ["2024-02.nc"])
|
|
56
56
|
atlas.info(store)
|
|
57
57
|
```
|
|
58
58
|
|
|
@@ -15,8 +15,20 @@ atlas create /data/nc /data/collection
|
|
|
15
15
|
## What happens
|
|
16
16
|
|
|
17
17
|
`create` collects every file that matches `.nc`, `.nc4`, `.cdf`, or `.netcdf`.
|
|
18
|
-
|
|
19
|
-
`2024-01.nc` becomes the
|
|
18
|
+
The scan **descends into every subdirectory**. It **sorts** the result, and
|
|
19
|
+
writes one dataset per file, named after the file. `2024-01.nc` becomes the
|
|
20
|
+
dataset `2024-01.nc`. The suffix is part of the name, so `jan.nc` and
|
|
21
|
+
`jan.nc4` are two datasets.
|
|
22
|
+
|
|
23
|
+
Pass `recursive=False`, or `--no-recursive`, to scan the top directory alone:
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
atlas.create("/data/nc", dest, recursive=False)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
atlas create /data/nc /data/collection --no-recursive
|
|
31
|
+
```
|
|
20
32
|
|
|
21
33
|
The sort matters. An ordinal comes from the write order, so a sorted ingest
|
|
22
34
|
makes it reproducible. Rebuild the same directory, and every dataset lands at
|
|
@@ -25,8 +37,8 @@ the same position.
|
|
|
25
37
|
Check what the call picks up before you run it:
|
|
26
38
|
|
|
27
39
|
```python
|
|
28
|
-
atlas.find_netcdf_files("/data/nc")
|
|
29
|
-
atlas.find_netcdf_files("/data/nc", recursive=
|
|
40
|
+
atlas.find_netcdf_files("/data/nc") # sorted, and recursive
|
|
41
|
+
atlas.find_netcdf_files("/data/nc", recursive=False) # the top level alone
|
|
30
42
|
```
|
|
31
43
|
|
|
32
44
|
## All or nothing
|
|
@@ -39,7 +51,7 @@ That is what you usually want. When it is not:
|
|
|
39
51
|
|
|
40
52
|
```python
|
|
41
53
|
result = atlas.create("/data/nc", dest, on_error="skip")
|
|
42
|
-
result["written"] # ['2024-01', '2024-03']
|
|
54
|
+
result["written"] # ['2024-01.nc', '2024-03.nc']
|
|
43
55
|
result["skipped"] # [{'file': '.../2024-02.nc', 'error': '...'}]
|
|
44
56
|
```
|
|
45
57
|
|
|
@@ -51,6 +63,78 @@ A skipped file leaves no trace in the collection. The writer moves to the next
|
|
|
51
63
|
one. The CLI exits `1` when it skipped anything, so a pipeline sees that. It
|
|
52
64
|
still writes the collection.
|
|
53
65
|
|
|
66
|
+
## One bad array, not one bad file
|
|
67
|
+
|
|
68
|
+
`on_error` works at the granularity of a file. One variable of an unsupported
|
|
69
|
+
dtype therefore costs the whole dataset. `on_unsupported="skip"` narrows that
|
|
70
|
+
to the array:
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
result = atlas.create("/data/nc", dest, on_unsupported="skip")
|
|
74
|
+
result["skipped_arrays"]
|
|
75
|
+
# [{'array': 'flag', 'dtype': 'bool', 'error': '...', 'dataset': '2024-01.nc'}]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
atlas create /data/nc /data/collection --skip-unsupported
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The rest of the dataset lands as usual: every other array, every attribute,
|
|
83
|
+
and the dataset itself. The skipped name is absent from the schema, so no
|
|
84
|
+
empty array stands in for it.
|
|
85
|
+
|
|
86
|
+
Atlas resolves every dtype before it defines the first array. A skip therefore
|
|
87
|
+
never leaves a half-written array behind.
|
|
88
|
+
|
|
89
|
+
The two settings compose. `--skip-unsupported` handles the array atlas cannot
|
|
90
|
+
store. `--skip-errors` handles the file that fails for any other reason.
|
|
91
|
+
|
|
92
|
+
See [Supported dtypes](dtypes.md) for what atlas can store.
|
|
93
|
+
|
|
94
|
+
## The log file
|
|
95
|
+
|
|
96
|
+
Both kinds of skip go to a log file, with the reason:
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```text
|
|
103
|
+
2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
|
|
104
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/broken.nc: ValueError: did not find a match ...
|
|
105
|
+
2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
|
|
106
|
+
2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 1 file(s) and 1 array(s)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Each line names the file, so a thousand-file ingest stays readable. The file
|
|
110
|
+
opens in append mode, so a repeat run adds to it.
|
|
111
|
+
|
|
112
|
+
From the library:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
import atlas
|
|
116
|
+
|
|
117
|
+
atlas.log_to_file("ingest.log")
|
|
118
|
+
atlas.create("/data/nc", dest, on_unsupported="skip")
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Atlas logs to the `atlas` logger and attaches no handler of its own, so attach
|
|
122
|
+
your own to send the records somewhere else:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
import logging
|
|
126
|
+
|
|
127
|
+
logging.getLogger("atlas").addHandler(logging.StreamHandler())
|
|
128
|
+
logging.getLogger("atlas").setLevel(logging.INFO)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`log_to_file` also captures Python warnings, such as the one about missing
|
|
132
|
+
string cells. That moves them off stderr, because `logging.captureWarnings` is
|
|
133
|
+
process-wide.
|
|
134
|
+
|
|
135
|
+
The Rust core logs separately, through `tracing`. `atlas.init_tracing()` sends
|
|
136
|
+
that stream to stderr. See [Installation](../installation.md).
|
|
137
|
+
|
|
54
138
|
## Chunking and memory
|
|
55
139
|
|
|
56
140
|
These are one decision. The blocks a file *reads* in are the chunks it
|
|
@@ -114,12 +198,12 @@ alone. A one-chunk array reads whole, or not at all.
|
|
|
114
198
|
Confirm what landed:
|
|
115
199
|
|
|
116
200
|
```python
|
|
117
|
-
arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01")["arrays"]}
|
|
201
|
+
arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01.nc")["arrays"]}
|
|
118
202
|
arrays["temperature"]["chunk_shape"]
|
|
119
203
|
```
|
|
120
204
|
|
|
121
205
|
```bash
|
|
122
|
-
atlas show dest 2024-01 | grep _ChunkShape
|
|
206
|
+
atlas show dest 2024-01.nc | grep _ChunkShape
|
|
123
207
|
```
|
|
124
208
|
|
|
125
209
|
### The writer's own memory
|
|
@@ -157,12 +241,19 @@ turns it off.
|
|
|
157
241
|
| Situation | Result |
|
|
158
242
|
|---|---|
|
|
159
243
|
| Directory holds no NetCDF files | `AtlasError` |
|
|
160
|
-
| Two files share a
|
|
161
|
-
| A variable has a dtype atlas cannot store | `AtlasError
|
|
244
|
+
| Two files share a name | `AtlasError`. A dataset name must be unique |
|
|
245
|
+
| A variable has a dtype atlas cannot store | `AtlasError`, or one skipped array under `--skip-unsupported` |
|
|
246
|
+
| Any other bad file | `AtlasError`, or one skipped file under `--skip-errors` |
|
|
162
247
|
| Destination URL cannot be resolved | `SourceError` |
|
|
163
248
|
|
|
164
|
-
Two files with one
|
|
165
|
-
the name `jan`.
|
|
249
|
+
Two files with one name surprise people. `a/jan.nc` and `b/jan.nc` both want
|
|
250
|
+
the name `jan.nc`. The scan descends by default, so a tree of monthly
|
|
251
|
+
directories hits this often. A dataset name carries no directory, because a
|
|
252
|
+
name may hold no `/`.
|
|
253
|
+
|
|
254
|
+
Three ways out. Rename the files. Ingest each subdirectory into its own
|
|
255
|
+
collection. Or pass `on_error="skip"`, which keeps the first file and reports
|
|
256
|
+
the second.
|
|
166
257
|
|
|
167
258
|
For the dtype rules, see [Supported dtypes](dtypes.md).
|
|
168
259
|
|
|
@@ -62,7 +62,7 @@ fill of the array, and not as an attribute.
|
|
|
62
62
|
Check what landed:
|
|
63
63
|
|
|
64
64
|
```python
|
|
65
|
-
arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01")["arrays"]}
|
|
65
|
+
arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01.nc")["arrays"]}
|
|
66
66
|
arrays["temperature"]["fill_value"]
|
|
67
67
|
```
|
|
68
68
|
|
|
@@ -86,7 +86,10 @@ array data, not to the attributes. They come through unchanged:
|
|
|
86
86
|
| `list[...]`, `fixed_size_list[..., N]` | Not exposed yet |
|
|
87
87
|
|
|
88
88
|
A NetCDF file with a boolean variable fails the ingest. Store it as `uint8`,
|
|
89
|
-
and write the convention down.
|
|
89
|
+
and write the convention down. To keep the rest of that file, pass
|
|
90
|
+
`--skip-unsupported`, or `on_unsupported="skip"`. That leaves out the one array
|
|
91
|
+
and lands everything else. See
|
|
92
|
+
[Creating a collection](creating.md#one-bad-array-not-one-bad-file).
|
|
90
93
|
|
|
91
94
|
All four work as an **attribute** value, which is where they usually appear.
|
|
92
95
|
|
|
@@ -100,7 +103,7 @@ Some xarray attributes are no scalar. A nested dict, a ragged list, and a numpy
|
|
|
100
103
|
array each encode as JSON on the way in. `describe` decodes them again:
|
|
101
104
|
|
|
102
105
|
```python
|
|
103
|
-
atlas.describe(collection, "2024-01")["attributes"]
|
|
106
|
+
atlas.describe(collection, "2024-01.nc")["attributes"]
|
|
104
107
|
# {'month': 1, 'bounds': [1.0, 2.0], 'nested': {'a': 1}}
|
|
105
108
|
```
|
|
106
109
|
|
|
@@ -7,7 +7,7 @@ fetches array data, so a catalogue of ten thousand datasets costs one request.
|
|
|
7
7
|
|
|
8
8
|
```python
|
|
9
9
|
atlas.list_datasets("/data/collection")
|
|
10
|
-
# ['2024-01', '2024-02', '2024-03']
|
|
10
|
+
# ['2024-01.nc', '2024-02.nc', '2024-03.nc']
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
```bash
|
|
@@ -63,11 +63,11 @@ Use `describe` for the statistics of one dataset on its own.
|
|
|
63
63
|
## One dataset in detail: `describe` and `show`
|
|
64
64
|
|
|
65
65
|
```python
|
|
66
|
-
atlas.describe("/data/collection", "2024-01")
|
|
66
|
+
atlas.describe("/data/collection", "2024-01.nc")
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
atlas show /data/collection 2024-01
|
|
70
|
+
atlas show /data/collection 2024-01.nc
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
The CLI prints it like `ncdump -h`. The library returns the structure. Both
|
|
@@ -78,7 +78,7 @@ attributes, the coordinate flag, and the statistics.
|
|
|
78
78
|
`name` is a dataset name, or the NetCDF path the dataset came from:
|
|
79
79
|
|
|
80
80
|
```python
|
|
81
|
-
atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01"
|
|
81
|
+
atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01.nc"
|
|
82
82
|
```
|
|
83
83
|
|
|
84
84
|
### Statistics
|
|
@@ -117,7 +117,7 @@ The ingest records which variables were xarray coordinates, and reports them
|
|
|
117
117
|
back:
|
|
118
118
|
|
|
119
119
|
```python
|
|
120
|
-
detail = atlas.describe(collection, "2024-01")
|
|
120
|
+
detail = atlas.describe(collection, "2024-01.nc")
|
|
121
121
|
detail["coordinates"] # ['lat', 'lon']
|
|
122
122
|
[a["name"] for a in detail["arrays"] if a["is_coordinate"]]
|
|
123
123
|
```
|
|
@@ -4,10 +4,10 @@ Python reads the **metadata** of a collection. It reads no array value.
|
|
|
4
4
|
|
|
5
5
|
```python
|
|
6
6
|
atlas.list_datasets(collection) # ✓
|
|
7
|
-
atlas.describe(collection, "jan")
|
|
7
|
+
atlas.describe(collection, "jan.nc") # ✓ types, shapes, attrs, stats
|
|
8
8
|
atlas.info(collection) # ✓ counts, size, collection stats
|
|
9
9
|
|
|
10
|
-
atlas.read_array(collection, "jan", "t")
|
|
10
|
+
atlas.read_array(collection, "jan.nc", "t") # ✗ does not exist
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
## Why
|
|
@@ -27,7 +27,7 @@ It is enough to build a catalogue, to check an ingest, or to choose the
|
|
|
27
27
|
datasets worth a fetch:
|
|
28
28
|
|
|
29
29
|
```python
|
|
30
|
-
detail = atlas.describe(collection, "2024-01")
|
|
30
|
+
detail = atlas.describe(collection, "2024-01.nc")
|
|
31
31
|
|
|
32
32
|
detail["dimensions"] # {'lat': 4, 'lon': 6}
|
|
33
33
|
detail["coordinates"] # ['lat', 'lon']
|
|
@@ -99,7 +99,7 @@ simpler than anything else here.
|
|
|
99
99
|
reader opens it:
|
|
100
100
|
|
|
101
101
|
```python
|
|
102
|
-
start, end = atlas.describe(collection, "2024-01")["segment_range"]
|
|
102
|
+
start, end = atlas.describe(collection, "2024-01.nc")["segment_range"]
|
|
103
103
|
blob = open(f"{collection}/data.atlas", "rb").read()[start:end]
|
|
104
104
|
open("2024-01.af", "wb").write(blob)
|
|
105
105
|
```
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# Removing datasets
|
|
2
2
|
|
|
3
3
|
```python
|
|
4
|
-
atlas.remove("/data/collection", ["2024-02", "2024-03"])
|
|
4
|
+
atlas.remove("/data/collection", ["2024-02.nc", "2024-03.nc"])
|
|
5
5
|
```
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
atlas rm /data/collection 2024-02 2024-03
|
|
8
|
+
atlas rm /data/collection 2024-02.nc 2024-03.nc
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
One call, whatever the number of datasets. A name is a dataset name, or the
|
|
@@ -66,7 +66,7 @@ mask holds a sorted set of ordinals.
|
|
|
66
66
|
The same holds on the command line, up to the argument limit of your shell:
|
|
67
67
|
|
|
68
68
|
```bash
|
|
69
|
-
atlas rm /data/collection 2024-01 2024-02 2024-03
|
|
69
|
+
atlas rm /data/collection 2024-01.nc 2024-02.nc 2024-03.nc
|
|
70
70
|
```
|
|
71
71
|
|
|
72
72
|
For a list too long for one command line, call `atlas.remove` from Python.
|
|
@@ -83,8 +83,8 @@ atlas.remove(collection, ["nope"])
|
|
|
83
83
|
`missing_ok` reports it instead:
|
|
84
84
|
|
|
85
85
|
```python
|
|
86
|
-
result = atlas.remove(collection, ["2024-01", "nope"], missing_ok=True)
|
|
87
|
-
result["removed"] # ['2024-01']
|
|
86
|
+
result = atlas.remove(collection, ["2024-01.nc", "nope"], missing_ok=True)
|
|
87
|
+
result["removed"] # ['2024-01.nc']
|
|
88
88
|
result["missing"] # ['nope']
|
|
89
89
|
```
|
|
90
90
|
|
|
@@ -10,9 +10,9 @@ atlas create /data/nc /data/collection
|
|
|
10
10
|
|
|
11
11
|
```text
|
|
12
12
|
$ atlas ls /data/collection
|
|
13
|
-
2024-01
|
|
14
|
-
2024-02
|
|
15
|
-
2024-03
|
|
13
|
+
2024-01.nc
|
|
14
|
+
2024-02.nc
|
|
15
|
+
2024-03.nc
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
## The shape of it
|
|
@@ -60,8 +60,8 @@ atlas.list_datasets("s3://bucket/2024", region="eu-west-1")
|
|
|
60
60
|
## What `show` gives you
|
|
61
61
|
|
|
62
62
|
```text
|
|
63
|
-
$ atlas show /data/collection 2024-01
|
|
64
|
-
dataset 2024-01 {
|
|
63
|
+
$ atlas show /data/collection 2024-01.nc
|
|
64
|
+
dataset 2024-01.nc {
|
|
65
65
|
dimensions:
|
|
66
66
|
lat = 4 ;
|
|
67
67
|
lon = 6 ;
|