atlas-python 0.16.0__tar.gz → 0.16.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {atlas_python-0.16.0 → atlas_python-0.16.2}/Cargo.lock +2 -2
  2. {atlas_python-0.16.0 → atlas_python-0.16.2}/Cargo.toml +1 -1
  3. {atlas_python-0.16.0 → atlas_python-0.16.2}/PKG-INFO +21 -10
  4. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/Cargo.toml +1 -1
  5. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/README.md +20 -9
  6. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/cli.md +62 -13
  7. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/examples.md +2 -2
  8. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/cloud-storage.md +2 -2
  9. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/creating.md +102 -11
  10. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/dtypes.md +6 -3
  11. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/inspecting.md +5 -5
  12. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/reading-data.md +4 -4
  13. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/guides/removing.md +5 -5
  14. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/index.md +5 -5
  15. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/installation.md +41 -0
  16. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/quickstart.md +13 -13
  17. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/reference/api.md +4 -0
  18. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/docs/vs-zarr-netcdf.md +1 -1
  19. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/examples/01_library.py +4 -4
  20. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/examples/02_object_store.py +2 -2
  21. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/conftest.py +20 -1
  22. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_cli.py +126 -17
  23. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_ops.py +209 -62
  24. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/test_source.py +3 -3
  25. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/format.md +10 -3
  26. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/python.md +35 -5
  27. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/write-path.md +1 -0
  28. {atlas_python-0.16.0 → atlas_python-0.16.2}/pyproject.toml +1 -1
  29. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/__init__.py +2 -0
  30. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/__init__.pyi +54 -12
  31. atlas_python-0.16.2/python/atlas/__main__.py +14 -0
  32. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_cli.py +54 -10
  33. atlas_python-0.16.2/python/atlas/_log.py +82 -0
  34. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_ops.py +73 -17
  35. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/xarray.py +64 -11
  36. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/footer.rs +39 -3
  37. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/writer/mod.rs +10 -0
  38. {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/cross_fixture.rs +6 -6
  39. {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/integration.rs +25 -0
  40. {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-python-docs.yaml +0 -0
  41. {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-python-release.yaml +0 -0
  42. {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/atlas-rust-release.yaml +0 -0
  43. {atlas_python-0.16.0 → atlas_python-0.16.2}/.github/workflows/ci.yaml +0 -0
  44. {atlas_python-0.16.0 → atlas_python-0.16.2}/.gitignore +0 -0
  45. {atlas_python-0.16.0 → atlas_python-0.16.2}/CONTRIBUTING.md +0 -0
  46. {atlas_python-0.16.0 → atlas_python-0.16.2}/LICENSE +0 -0
  47. {atlas_python-0.16.0 → atlas_python-0.16.2}/README.md +0 -0
  48. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/.python-version +0 -0
  49. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/LICENSE +0 -0
  50. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/mkdocs.yml +0 -0
  51. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/attr.rs +0 -0
  52. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/dtype.rs +0 -0
  53. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/error.rs +0 -0
  54. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/lib.rs +0 -0
  55. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/logging.rs +0 -0
  56. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/reader.rs +0 -0
  57. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/runtime.rs +0 -0
  58. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/source.rs +0 -0
  59. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/src/writer.rs +0 -0
  60. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/GL_PR_BO_JLKU.nc +0 -0
  61. {atlas_python-0.16.0 → atlas_python-0.16.2}/atlas-python/tests/make_fixture.py +0 -0
  62. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/README.md +0 -0
  63. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/architecture.md +0 -0
  64. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/data-model.md +0 -0
  65. {atlas_python-0.16.0 → atlas_python-0.16.2}/docs/read-path.md +0 -0
  66. {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/lifecycle.rs +0 -0
  67. {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/sensor_fleet.rs +0 -0
  68. {atlas_python-0.16.0 → atlas_python-0.16.2}/examples/weather_store.rs +0 -0
  69. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/_source.py +0 -0
  70. {atlas_python-0.16.0 → atlas_python-0.16.2}/python/atlas/py.typed +0 -0
  71. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/config.rs +0 -0
  72. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/error.rs +0 -0
  73. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/mask.rs +0 -0
  74. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/mod.rs +0 -0
  75. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/format/segment_store.rs +0 -0
  76. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/lib.rs +0 -0
  77. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/reader/mod.rs +0 -0
  78. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/array.rs +0 -0
  79. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/attr.rs +0 -0
  80. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/dtype.rs +0 -0
  81. {atlas_python-0.16.0 → atlas_python-0.16.2}/src/schema/mod.rs +0 -0
  82. {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/fixtures/from_python/data.atlas +0 -0
  83. {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/fixtures/golden_v1/data.atlas +0 -0
  84. {atlas_python-0.16.0 → atlas_python-0.16.2}/tests/golden.rs +0 -0
@@ -71,7 +71,7 @@ dependencies = [
71
71
 
72
72
  [[package]]
73
73
  name = "atlas-python"
74
- version = "0.16.0"
74
+ version = "0.16.2"
75
75
  dependencies = [
76
76
  "atlas-rust",
77
77
  "ndarray",
@@ -86,7 +86,7 @@ dependencies = [
86
86
 
87
87
  [[package]]
88
88
  name = "atlas-rust"
89
- version = "0.16.0"
89
+ version = "0.16.2"
90
90
  dependencies = [
91
91
  "array-format",
92
92
  "async-trait",
@@ -5,7 +5,7 @@ members = ["atlas-python"]
5
5
  # Registry name on crates.io (`atlas` is taken). The library is still
6
6
  # imported as `atlas` — see `[lib]` below.
7
7
  name = "atlas-rust"
8
- version = "0.16.0"
8
+ version = "0.16.2"
9
9
  edition = "2024"
10
10
  description = "Single-file immutable store for thousands of N-dimensional datasets, local or on object storage."
11
11
  license = "Apache-2.0"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: atlas-python
3
- Version: 0.16.0
3
+ Version: 0.16.2
4
4
  Classifier: Development Status :: 3 - Alpha
5
5
  Classifier: Intended Audience :: Science/Research
6
6
  Classifier: Programming Language :: Python :: 3
@@ -39,11 +39,14 @@ pip install atlas-python
39
39
  ```bash
40
40
  atlas create /data/nc /data/collection
41
41
  atlas ls /data/collection
42
- atlas show /data/collection 2024-01
42
+ atlas show /data/collection 2024-01.nc
43
43
  atlas info /data/collection
44
- atlas rm /data/collection 2024-02 2024-03
44
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
45
45
  ```
46
46
 
47
+ `python -m atlas` runs the same command without a PATH lookup, for a shell that
48
+ cannot find `atlas`.
49
+
47
50
  | Extra | Install | Adds |
48
51
  |---|---|---|
49
52
  | cloud | `pip install "atlas-python[cloud]"` | S3 / GCS / Azure / HTTP via [obstore](https://github.com/developmentseed/obstore) |
@@ -58,10 +61,10 @@ The same five as a library:
58
61
  import atlas
59
62
 
60
63
  atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
61
- atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
62
- atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
64
+ atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
65
+ atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
63
66
  atlas.info("/data/collection") # counts, size, codec, statistics
64
- atlas.remove("/data/collection", ["2024-02"]) # updates the mask
67
+ atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
65
68
  ```
66
69
 
67
70
  Every one takes a local path, a URL, or an obstore handle:
@@ -94,8 +97,8 @@ request.
94
97
  ## What `show` gives you
95
98
 
96
99
  ```text
97
- $ atlas show /data/collection 2024-01
98
- dataset 2024-01 {
100
+ $ atlas show /data/collection 2024-01.nc
101
+ dataset 2024-01.nc {
99
102
  dimensions:
100
103
  lat = 4 ;
101
104
  lon = 6 ;
@@ -120,8 +123,9 @@ command gives the same content as a structure.
120
123
 
121
124
  ## Ingest
122
125
 
123
- `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
124
- them, and writes one dataset per file, named after the stem. Each coordinate
126
+ `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`, and
127
+ descends into every subdirectory. It sorts them, and writes one dataset per
128
+ file, named after the file. Each coordinate
125
129
  and data variable becomes an array. Each variable attribute becomes a per-array
126
130
  attribute. `_FillValue` becomes the fill of the array.
127
131
 
@@ -134,6 +138,13 @@ Nothing at the destination is readable until every file lands. A failure
134
138
  part-way leaves no collection, and not a partial one. `on_error="skip"`, or
135
139
  `--skip-errors`, trades that for progress.
136
140
 
141
+ Atlas cannot store every numpy dtype, and `bool` is the common case. One such
142
+ variable fails the whole file by default. `on_unsupported="skip"`, or
143
+ `--skip-unsupported`, leaves out that one array and lands the rest.
144
+
145
+ `--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
146
+ warning to a file, each with its reason and the file it came from.
147
+
137
148
  ## dtypes
138
149
 
139
150
  | numpy | atlas |
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "atlas-python"
3
- version = "0.16.0"
3
+ version = "0.16.2"
4
4
  edition = "2021"
5
5
  readme = "README.md"
6
6
 
@@ -11,11 +11,14 @@ pip install atlas-python
11
11
  ```bash
12
12
  atlas create /data/nc /data/collection
13
13
  atlas ls /data/collection
14
- atlas show /data/collection 2024-01
14
+ atlas show /data/collection 2024-01.nc
15
15
  atlas info /data/collection
16
- atlas rm /data/collection 2024-02 2024-03
16
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
17
17
  ```
18
18
 
19
+ `python -m atlas` runs the same command without a PATH lookup, for a shell that
20
+ cannot find `atlas`.
21
+
19
22
  | Extra | Install | Adds |
20
23
  |---|---|---|
21
24
  | cloud | `pip install "atlas-python[cloud]"` | S3 / GCS / Azure / HTTP via [obstore](https://github.com/developmentseed/obstore) |
@@ -30,10 +33,10 @@ The same five as a library:
30
33
  import atlas
31
34
 
32
35
  atlas.create("/data/nc", "/data/collection") # from a directory of NetCDF files
33
- atlas.list_datasets("/data/collection") # ['2024-01', '2024-02', '2024-03']
34
- atlas.describe("/data/collection", "2024-01") # types, shapes, attrs, statistics
36
+ atlas.list_datasets("/data/collection") # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
37
+ atlas.describe("/data/collection", "2024-01.nc") # types, shapes, attrs, statistics
35
38
  atlas.info("/data/collection") # counts, size, codec, statistics
36
- atlas.remove("/data/collection", ["2024-02"]) # updates the mask
39
+ atlas.remove("/data/collection", ["2024-02.nc"]) # updates the mask
37
40
  ```
38
41
 
39
42
  Every one takes a local path, a URL, or an obstore handle:
@@ -66,8 +69,8 @@ request.
66
69
  ## What `show` gives you
67
70
 
68
71
  ```text
69
- $ atlas show /data/collection 2024-01
70
- dataset 2024-01 {
72
+ $ atlas show /data/collection 2024-01.nc
73
+ dataset 2024-01.nc {
71
74
  dimensions:
72
75
  lat = 4 ;
73
76
  lon = 6 ;
@@ -92,8 +95,9 @@ command gives the same content as a structure.
92
95
 
93
96
  ## Ingest
94
97
 
95
- `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`. It sorts
96
- them, and writes one dataset per file, named after the stem. Each coordinate
98
+ `create` scans a directory for `.nc`, `.nc4`, `.cdf`, and `.netcdf`, and
99
+ descends into every subdirectory. It sorts them, and writes one dataset per
100
+ file, named after the file. Each coordinate
97
101
  and data variable becomes an array. Each variable attribute becomes a per-array
98
102
  attribute. `_FillValue` becomes the fill of the array.
99
103
 
@@ -106,6 +110,13 @@ Nothing at the destination is readable until every file lands. A failure
106
110
  part-way leaves no collection, and not a partial one. `on_error="skip"`, or
107
111
  `--skip-errors`, trades that for progress.
108
112
 
113
+ Atlas cannot store every numpy dtype, and `bool` is the common case. One such
114
+ variable fails the whole file by default. `on_unsupported="skip"`, or
115
+ `--skip-unsupported`, leaves out that one array and lands the rest.
116
+
117
+ `--log-file PATH`, or `atlas.log_to_file(path)`, appends every error and
118
+ warning to a file, each with its reason and the file it came from.
119
+
109
120
  ## dtypes
110
121
 
111
122
  | numpy | atlas |
@@ -3,6 +3,10 @@
3
3
  `pip install atlas-python` puts `atlas` on your PATH. Five subcommands, one per
4
4
  operation.
5
5
 
6
+ `python -m atlas` runs the same command without a PATH lookup. Use it in a
7
+ container, in a CI job, or when the shell cannot find `atlas`. See
8
+ [Installation](installation.md#the-atlas-command-is-not-found).
9
+
6
10
  ```text
7
11
  atlas create <netcdf-dir> <collection> build a collection
8
12
  atlas rm <collection> <name>... remove datasets
@@ -16,8 +20,27 @@ A `<collection>` is a local path or a URL: `s3://bucket/prefix`, `gs://...`,
16
20
  `pip install "atlas-python[cloud]"`. See
17
21
  [Cloud storage](guides/cloud-storage.md).
18
22
 
19
- Every subcommand takes `--json`, and the remote flags `--region`,
20
- `--endpoint`, and `--anonymous`.
23
+ Every subcommand takes `--json` and `--log-file PATH`, plus the remote flags
24
+ `--region`, `--endpoint`, and `--anonymous`.
25
+
26
+ ## Logging
27
+
28
+ `--log-file PATH` appends every error and warning to a file, with the reason:
29
+
30
+ ```bash
31
+ $ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
32
+ $ cat ingest.log
33
+ 2026-09-01 14:30:41 INFO atlas.cli: atlas 0.16.2: create /data/nc ...
34
+ 2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
35
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
36
+ 2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 0 file(s) and 1 array(s)
37
+ ```
38
+
39
+ The file opens in append mode. Each line names the file it came from, so a
40
+ large ingest stays readable.
41
+
42
+ This is separate from `ATLAS_LOG`, which turns on the Rust `tracing` stream to
43
+ stderr.
21
44
 
22
45
  ## create
23
46
 
@@ -25,25 +48,51 @@ Every subcommand takes `--json`, and the remote flags `--region`,
25
48
  atlas create /data/nc /data/collection
26
49
  ```
27
50
 
28
- Each NetCDF file becomes one dataset, named after its stem. `2024-01.nc`
29
- becomes `2024-01`. The files land in sorted order, which makes the ordinals of
30
- a collection reproducible.
51
+ The scan descends into every subdirectory. Each NetCDF file becomes one
52
+ dataset, named after the file. `2024-01.nc` becomes `2024-01.nc`, suffix and
53
+ all. A name carries no directory, so two files of one name in two
54
+ subdirectories collide. The files land in sorted order, which makes the
55
+ ordinals of a collection reproducible.
31
56
 
32
57
  Nothing at the destination is readable until every file lands, with the footer.
33
58
  A failure part-way leaves no collection, and not a partial one.
34
59
 
35
60
  | Flag | Effect |
36
61
  |---|---|
37
- | `-r`, `--recursive` | Descend into subdirectories |
62
+ | `--no-recursive` | Scan the top directory alone. The scan descends by default |
63
+ | `-r`, `--recursive` | Accepted for compatibility. The scan already descends |
38
64
  | `--codec {zstd,lz4,none}` | Block compression. Default `zstd` |
39
65
  | `--chunk-size SIZE` | Block size to aim for. Default `128MiB` |
40
66
  | `--open-chunks MODE` | How files are read: `auto`, `native`, `none`, or a JSON dict |
41
67
  | `--chunks JSON` | Override the stored chunk shape, `'{"temperature": [64, 64]}'` |
42
68
  | `--skip-errors` | Skip files that fail instead of abandoning the collection |
69
+ | `--skip-unsupported` | Leave out an array of an unsupported dtype, and keep the rest of the dataset |
43
70
  | `-q`, `--quiet` | Do not list a file as it lands |
44
71
 
45
72
  Progress goes to stderr, so a pipe still reads stdout.
46
73
 
74
+ ### Unsupported dtypes
75
+
76
+ Atlas cannot store every numpy dtype. `bool` is the common case. By default one
77
+ such variable fails the whole file:
78
+
79
+ ```bash
80
+ $ atlas create /data/nc /data/collection
81
+ atlas: /data/nc/buoy.nc: numpy dtype dtype('bool') is not supported by atlas ...
82
+ ```
83
+
84
+ `--skip-unsupported` narrows that to the one array:
85
+
86
+ ```bash
87
+ $ atlas create /data/nc /data/collection --skip-unsupported
88
+ skipped array buoy/flag (bool)
89
+ 1 dataset(s) written to /data/collection
90
+ ```
91
+
92
+ Every other array of that dataset lands, with its attributes. `--json` reports
93
+ the skipped arrays under `skipped_arrays`, with the dataset, the dtype, and the
94
+ reason. See [Supported dtypes](guides/dtypes.md).
95
+
47
96
  ### Large files
48
97
 
49
98
  Each file reads in dask blocks. A file far larger than memory therefore
@@ -79,7 +128,7 @@ reads. Each misaligned block then costs a read-modify-write. Use
79
128
 
80
129
  ```bash
81
130
  # One collection from a tree of monthly directories, tolerating bad files
82
- atlas create /data/nc /data/collection --recursive --skip-errors
131
+ atlas create /data/nc /data/collection --skip-errors
83
132
 
84
133
  # A big grid, chunked for selective reads, straight to a bucket
85
134
  atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
@@ -88,7 +137,7 @@ atlas create /data/nc s3://bucket/2024 --chunk-size 64MiB --region eu-west-1
88
137
  ## rm
89
138
 
90
139
  ```bash
91
- atlas rm /data/collection 2024-02 2024-03
140
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
92
141
  ```
93
142
 
94
143
  This removes several datasets in one call. A name is a dataset name, or the
@@ -114,9 +163,9 @@ error.
114
163
 
115
164
  ```bash
116
165
  $ atlas ls /data/collection
117
- 2024-01
118
- 2024-02
119
- 2024-03
166
+ 2024-01.nc
167
+ 2024-02.nc
168
+ 2024-03.nc
120
169
  ```
121
170
 
122
171
  One name per line, in write order. A removed dataset does not appear. This
@@ -131,8 +180,8 @@ atlas ls /data/collection --json | jq . # as a JSON array
131
180
  ## show
132
181
 
133
182
  ```bash
134
- $ atlas show /data/collection 2024-01
135
- dataset 2024-01 {
183
+ $ atlas show /data/collection 2024-01.nc
184
+ dataset 2024-01.nc {
136
185
  dimensions:
137
186
  lat = 4 ;
138
187
  lon = 6 ;
@@ -19,8 +19,8 @@ python atlas-python/examples/01_library.py
19
19
  atlas create /data/nc /data/collection
20
20
  atlas ls /data/collection
21
21
  atlas info /data/collection
22
- atlas show /data/collection 2024-01
23
- atlas rm /data/collection 2024-02 2024-03
22
+ atlas show /data/collection 2024-01.nc
23
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
24
24
  ```
25
25
 
26
26
  See [The `atlas` command](cli.md).
@@ -51,8 +51,8 @@ store = obs.store.S3Store(
51
51
 
52
52
  atlas.create("/data/nc", store)
53
53
  atlas.list_datasets(store)
54
- atlas.describe(store, "2024-01")
55
- atlas.remove(store, ["2024-02"])
54
+ atlas.describe(store, "2024-01.nc")
55
+ atlas.remove(store, ["2024-02.nc"])
56
56
  atlas.info(store)
57
57
  ```
58
58
 
@@ -15,8 +15,20 @@ atlas create /data/nc /data/collection
15
15
  ## What happens
16
16
 
17
17
  `create` collects every file that matches `.nc`, `.nc4`, `.cdf`, or `.netcdf`.
18
- It **sorts** them, and writes one dataset per file, named after the file stem.
19
- `2024-01.nc` becomes the dataset `2024-01`.
18
+ The scan **descends into every subdirectory**. It **sorts** the result, and
19
+ writes one dataset per file, named after the file. `2024-01.nc` becomes the
20
+ dataset `2024-01.nc`. The suffix is part of the name, so `jan.nc` and
21
+ `jan.nc4` are two datasets.
22
+
23
+ Pass `recursive=False`, or `--no-recursive`, to scan the top directory alone:
24
+
25
+ ```python
26
+ atlas.create("/data/nc", dest, recursive=False)
27
+ ```
28
+
29
+ ```bash
30
+ atlas create /data/nc /data/collection --no-recursive
31
+ ```
20
32
 
21
33
  The sort matters. An ordinal comes from the write order, so a sorted ingest
22
34
  makes it reproducible. Rebuild the same directory, and every dataset lands at
@@ -25,8 +37,8 @@ the same position.
25
37
  Check what the call picks up before you run it:
26
38
 
27
39
  ```python
28
- atlas.find_netcdf_files("/data/nc") # sorted list of paths
29
- atlas.find_netcdf_files("/data/nc", recursive=True)
40
+ atlas.find_netcdf_files("/data/nc") # sorted, and recursive
41
+ atlas.find_netcdf_files("/data/nc", recursive=False) # the top level alone
30
42
  ```
31
43
 
32
44
  ## All or nothing
@@ -39,7 +51,7 @@ That is what you usually want. When it is not:
39
51
 
40
52
  ```python
41
53
  result = atlas.create("/data/nc", dest, on_error="skip")
42
- result["written"] # ['2024-01', '2024-03']
54
+ result["written"] # ['2024-01.nc', '2024-03.nc']
43
55
  result["skipped"] # [{'file': '.../2024-02.nc', 'error': '...'}]
44
56
  ```
45
57
 
@@ -51,6 +63,78 @@ A skipped file leaves no trace in the collection. The writer moves to the next
51
63
  one. The CLI exits `1` when it skipped anything, so a pipeline sees that. It
52
64
  still writes the collection.
53
65
 
66
+ ## One bad array, not one bad file
67
+
68
+ `on_error` works at the granularity of a file. One variable of an unsupported
69
+ dtype therefore costs the whole dataset. `on_unsupported="skip"` narrows that
70
+ to the array:
71
+
72
+ ```python
73
+ result = atlas.create("/data/nc", dest, on_unsupported="skip")
74
+ result["skipped_arrays"]
75
+ # [{'array': 'flag', 'dtype': 'bool', 'error': '...', 'dataset': '2024-01.nc'}]
76
+ ```
77
+
78
+ ```bash
79
+ atlas create /data/nc /data/collection --skip-unsupported
80
+ ```
81
+
82
+ The rest of the dataset lands as usual: every other array, every attribute,
83
+ and the dataset itself. The skipped name is absent from the schema, so no
84
+ empty array stands in for it.
85
+
86
+ Atlas resolves every dtype before it defines the first array. A skip therefore
87
+ never leaves a half-written array behind.
88
+
89
+ The two settings compose. `--skip-unsupported` handles the array atlas cannot
90
+ store. `--skip-errors` handles the file that fails for any other reason.
91
+
92
+ See [Supported dtypes](dtypes.md) for what atlas can store.
93
+
94
+ ## The log file
95
+
96
+ Both kinds of skip go to a log file, with the reason:
97
+
98
+ ```bash
99
+ atlas create /data/nc /data/collection --skip-unsupported --log-file ingest.log
100
+ ```
101
+
102
+ ```text
103
+ 2026-09-01 14:30:41 INFO atlas.ops: ingesting 2 file(s) into /data/collection
104
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/broken.nc: ValueError: did not find a match ...
105
+ 2026-09-01 14:30:41 WARNING atlas.ops: /data/nc/buoy.nc: skipped array 'flag' of dtype bool: numpy dtype dtype('bool') is not supported by atlas (supported: ...)
106
+ 2026-09-01 14:30:41 INFO atlas.ops: wrote 1 dataset(s); skipped 1 file(s) and 1 array(s)
107
+ ```
108
+
109
+ Each line names the file, so a thousand-file ingest stays readable. The file
110
+ opens in append mode, so a repeat run adds to it.
111
+
112
+ From the library:
113
+
114
+ ```python
115
+ import atlas
116
+
117
+ atlas.log_to_file("ingest.log")
118
+ atlas.create("/data/nc", dest, on_unsupported="skip")
119
+ ```
120
+
121
+ Atlas logs to the `atlas` logger and attaches no handler of its own, so attach
122
+ your own to send the records somewhere else:
123
+
124
+ ```python
125
+ import logging
126
+
127
+ logging.getLogger("atlas").addHandler(logging.StreamHandler())
128
+ logging.getLogger("atlas").setLevel(logging.INFO)
129
+ ```
130
+
131
+ `log_to_file` also captures Python warnings, such as the one about missing
132
+ string cells. That moves them off stderr, because `logging.captureWarnings` is
133
+ process-wide.
134
+
135
+ The Rust core logs separately, through `tracing`. `atlas.init_tracing()` sends
136
+ that stream to stderr. See [Installation](../installation.md).
137
+
54
138
  ## Chunking and memory
55
139
 
56
140
  These are one decision. The blocks a file *reads* in are the chunks it
@@ -114,12 +198,12 @@ alone. A one-chunk array reads whole, or not at all.
114
198
  Confirm what landed:
115
199
 
116
200
  ```python
117
- arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01")["arrays"]}
201
+ arrays = {a["name"]: a for a in atlas.describe(dest, "2024-01.nc")["arrays"]}
118
202
  arrays["temperature"]["chunk_shape"]
119
203
  ```
120
204
 
121
205
  ```bash
122
- atlas show dest 2024-01 | grep _ChunkShape
206
+ atlas show dest 2024-01.nc | grep _ChunkShape
123
207
  ```
124
208
 
125
209
  ### The writer's own memory
@@ -157,12 +241,19 @@ turns it off.
157
241
  | Situation | Result |
158
242
  |---|---|
159
243
  | Directory holds no NetCDF files | `AtlasError` |
160
- | Two files share a stem | `AtlasError`. A dataset name must be unique |
161
- | A variable has a dtype atlas cannot store | `AtlasError` (or skipped) |
244
+ | Two files share a name | `AtlasError`. A dataset name must be unique |
245
+ | A variable has a dtype atlas cannot store | `AtlasError`, or one skipped array under `--skip-unsupported` |
246
+ | Any other bad file | `AtlasError`, or one skipped file under `--skip-errors` |
162
247
  | Destination URL cannot be resolved | `SourceError` |
163
248
 
164
- Two files with one stem surprise people. `a/jan.nc` and `b/jan.nc` both want
165
- the name `jan`. Rename one, or ingest them into two collections.
249
+ Two files with one name surprise people. `a/jan.nc` and `b/jan.nc` both want
250
+ the name `jan.nc`. The scan descends by default, so a tree of monthly
251
+ directories hits this often. A dataset name carries no directory, because a
252
+ name may hold no `/`.
253
+
254
+ Three ways out. Rename the files. Ingest each subdirectory into its own
255
+ collection. Or pass `on_error="skip"`, which keeps the first file and reports
256
+ the second.
166
257
 
167
258
  For the dtype rules, see [Supported dtypes](dtypes.md).
168
259
 
@@ -62,7 +62,7 @@ fill of the array, and not as an attribute.
62
62
  Check what landed:
63
63
 
64
64
  ```python
65
- arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01")["arrays"]}
65
+ arrays = {a["name"]: a for a in atlas.describe(collection, "2024-01.nc")["arrays"]}
66
66
  arrays["temperature"]["fill_value"]
67
67
  ```
68
68
 
@@ -86,7 +86,10 @@ array data, not to the attributes. They come through unchanged:
86
86
  | `list[...]`, `fixed_size_list[..., N]` | Not exposed yet |
87
87
 
88
88
  A NetCDF file with a boolean variable fails the ingest. Store it as `uint8`,
89
- and write the convention down.
89
+ and write the convention down. To keep the rest of that file, pass
90
+ `--skip-unsupported`, or `on_unsupported="skip"`. That leaves out the one array
91
+ and lands everything else. See
92
+ [Creating a collection](creating.md#one-bad-array-not-one-bad-file).
90
93
 
91
94
  All four work as an **attribute** value, which is where they usually appear.
92
95
 
@@ -100,7 +103,7 @@ Some xarray attributes are no scalar. A nested dict, a ragged list, and a numpy
100
103
  array each encode as JSON on the way in. `describe` decodes them again:
101
104
 
102
105
  ```python
103
- atlas.describe(collection, "2024-01")["attributes"]
106
+ atlas.describe(collection, "2024-01.nc")["attributes"]
104
107
  # {'month': 1, 'bounds': [1.0, 2.0], 'nested': {'a': 1}}
105
108
  ```
106
109
 
@@ -7,7 +7,7 @@ fetches array data, so a catalogue of ten thousand datasets costs one request.
7
7
 
8
8
  ```python
9
9
  atlas.list_datasets("/data/collection")
10
- # ['2024-01', '2024-02', '2024-03']
10
+ # ['2024-01.nc', '2024-02.nc', '2024-03.nc']
11
11
  ```
12
12
 
13
13
  ```bash
@@ -63,11 +63,11 @@ Use `describe` for the statistics of one dataset on its own.
63
63
  ## One dataset in detail: `describe` and `show`
64
64
 
65
65
  ```python
66
- atlas.describe("/data/collection", "2024-01")
66
+ atlas.describe("/data/collection", "2024-01.nc")
67
67
  ```
68
68
 
69
69
  ```bash
70
- atlas show /data/collection 2024-01
70
+ atlas show /data/collection 2024-01.nc
71
71
  ```
72
72
 
73
73
  The CLI prints it like `ncdump -h`. The library returns the structure. Both
@@ -78,7 +78,7 @@ attributes, the coordinate flag, and the statistics.
78
78
  `name` is a dataset name, or the NetCDF path the dataset came from:
79
79
 
80
80
  ```python
81
- atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01"
81
+ atlas.describe(collection, "/data/nc/2024-01.nc") # same as "2024-01.nc"
82
82
  ```
83
83
 
84
84
  ### Statistics
@@ -117,7 +117,7 @@ The ingest records which variables were xarray coordinates, and reports them
117
117
  back:
118
118
 
119
119
  ```python
120
- detail = atlas.describe(collection, "2024-01")
120
+ detail = atlas.describe(collection, "2024-01.nc")
121
121
  detail["coordinates"] # ['lat', 'lon']
122
122
  [a["name"] for a in detail["arrays"] if a["is_coordinate"]]
123
123
  ```
@@ -4,10 +4,10 @@ Python reads the **metadata** of a collection. It reads no array value.
4
4
 
5
5
  ```python
6
6
  atlas.list_datasets(collection) # ✓
7
- atlas.describe(collection, "jan") # ✓ types, shapes, attrs, stats
7
+ atlas.describe(collection, "jan.nc") # ✓ types, shapes, attrs, stats
8
8
  atlas.info(collection) # ✓ counts, size, collection stats
9
9
 
10
- atlas.read_array(collection, "jan", "t") # ✗ does not exist
10
+ atlas.read_array(collection, "jan.nc", "t") # ✗ does not exist
11
11
  ```
12
12
 
13
13
  ## Why
@@ -27,7 +27,7 @@ It is enough to build a catalogue, to check an ingest, or to choose the
27
27
  datasets worth a fetch:
28
28
 
29
29
  ```python
30
- detail = atlas.describe(collection, "2024-01")
30
+ detail = atlas.describe(collection, "2024-01.nc")
31
31
 
32
32
  detail["dimensions"] # {'lat': 4, 'lon': 6}
33
33
  detail["coordinates"] # ['lat', 'lon']
@@ -99,7 +99,7 @@ simpler than anything else here.
99
99
  reader opens it:
100
100
 
101
101
  ```python
102
- start, end = atlas.describe(collection, "2024-01")["segment_range"]
102
+ start, end = atlas.describe(collection, "2024-01.nc")["segment_range"]
103
103
  blob = open(f"{collection}/data.atlas", "rb").read()[start:end]
104
104
  open("2024-01.af", "wb").write(blob)
105
105
  ```
@@ -1,11 +1,11 @@
1
1
  # Removing datasets
2
2
 
3
3
  ```python
4
- atlas.remove("/data/collection", ["2024-02", "2024-03"])
4
+ atlas.remove("/data/collection", ["2024-02.nc", "2024-03.nc"])
5
5
  ```
6
6
 
7
7
  ```bash
8
- atlas rm /data/collection 2024-02 2024-03
8
+ atlas rm /data/collection 2024-02.nc 2024-03.nc
9
9
  ```
10
10
 
11
11
  One call, whatever the number of datasets. A name is a dataset name, or the
@@ -66,7 +66,7 @@ mask holds a sorted set of ordinals.
66
66
  The same holds on the command line, up to the argument limit of your shell:
67
67
 
68
68
  ```bash
69
- atlas rm /data/collection 2024-01 2024-02 2024-03
69
+ atlas rm /data/collection 2024-01.nc 2024-02.nc 2024-03.nc
70
70
  ```
71
71
 
72
72
  For a list too long for one command line, call `atlas.remove` from Python.
@@ -83,8 +83,8 @@ atlas.remove(collection, ["nope"])
83
83
  `missing_ok` reports it instead:
84
84
 
85
85
  ```python
86
- result = atlas.remove(collection, ["2024-01", "nope"], missing_ok=True)
87
- result["removed"] # ['2024-01']
86
+ result = atlas.remove(collection, ["2024-01.nc", "nope"], missing_ok=True)
87
+ result["removed"] # ['2024-01.nc']
88
88
  result["missing"] # ['nope']
89
89
  ```
90
90
 
@@ -10,9 +10,9 @@ atlas create /data/nc /data/collection
10
10
 
11
11
  ```text
12
12
  $ atlas ls /data/collection
13
- 2024-01
14
- 2024-02
15
- 2024-03
13
+ 2024-01.nc
14
+ 2024-02.nc
15
+ 2024-03.nc
16
16
  ```
17
17
 
18
18
  ## The shape of it
@@ -60,8 +60,8 @@ atlas.list_datasets("s3://bucket/2024", region="eu-west-1")
60
60
  ## What `show` gives you
61
61
 
62
62
  ```text
63
- $ atlas show /data/collection 2024-01
64
- dataset 2024-01 {
63
+ $ atlas show /data/collection 2024-01.nc
64
+ dataset 2024-01.nc {
65
65
  dimensions:
66
66
  lat = 4 ;
67
67
  lon = 6 ;