yirgacheffe 1.4.1__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of yirgacheffe might be problematic. Click here for more details.
- {yirgacheffe-1.4.1/yirgacheffe.egg-info → yirgacheffe-1.6.0}/PKG-INFO +115 -56
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/README.md +114 -55
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/pyproject.toml +1 -1
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_area.py +10 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_datatypes.py +0 -1
- yirgacheffe-1.6.0/tests/test_nodata.py +64 -0
- yirgacheffe-1.6.0/tests/test_openers.py +165 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_operators.py +70 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_raster.py +6 -6
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_union.py +1 -1
- yirgacheffe-1.6.0/yirgacheffe/__init__.py +11 -0
- yirgacheffe-1.6.0/yirgacheffe/_core.py +136 -0
- yirgacheffe-1.6.0/yirgacheffe/constants.py +8 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/area.py +10 -3
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/base.py +46 -10
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/constant.py +18 -9
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/group.py +55 -17
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/h3layer.py +8 -1
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/rasters.py +42 -12
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/rescaled.py +11 -3
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/vectors.py +25 -16
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/operators.py +85 -21
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/window.py +27 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0/yirgacheffe.egg-info}/PKG-INFO +115 -56
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe.egg-info/SOURCES.txt +3 -0
- yirgacheffe-1.4.1/yirgacheffe/__init__.py +0 -14
- yirgacheffe-1.4.1/yirgacheffe/constants.py +0 -2
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/LICENSE +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/MANIFEST.in +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/setup.cfg +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_auto_windowing.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_base.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_constants.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_group.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_h3layer.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_intersection.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_multiband.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_optimisation.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_parallel_operators.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_pickle.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_rescaling.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_rounding.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_save_with_window.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_sum_with_window.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_uniform_area_layer.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_vectors.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/tests/test_window.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/_backends/__init__.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/_backends/enumeration.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/_backends/mlx.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/_backends/numpy.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/layers/__init__.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe/rounding.py +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe.egg-info/dependency_links.txt +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe.egg-info/entry_points.txt +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe.egg-info/requires.txt +0 -0
- {yirgacheffe-1.4.1 → yirgacheffe-1.6.0}/yirgacheffe.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: yirgacheffe
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Summary: Abstraction of gdal datasets for doing basic math operations
|
|
5
5
|
Author-email: Michael Dales <mwd24@cam.ac.uk>
|
|
6
6
|
License-Expression: ISC
|
|
@@ -53,12 +53,12 @@ The motivation for Yirgacheffe layers is to make working with gdal data slightly
|
|
|
53
53
|
For example, if we wanted to do a simple [Area of Habitat](https://github.com/quantifyearth/aoh-calculator/) calculation, whereby we find the pixels where a species resides by combining its range polygon, its habitat preferences, and its elevation preferences, the code would be like this:
|
|
54
54
|
|
|
55
55
|
```python
|
|
56
|
-
|
|
56
|
+
import yirgaceffe as yg
|
|
57
57
|
|
|
58
|
-
habitat_map =
|
|
59
|
-
elevation_map =
|
|
60
|
-
range_polygon =
|
|
61
|
-
area_per_pixel_map =
|
|
58
|
+
habitat_map = yg.read_raster("habitats.tif")
|
|
59
|
+
elevation_map = yg.read_raster('elevation.tif')
|
|
60
|
+
range_polygon = yg.read_shape_like('species123.geojson', like=habitat_map)
|
|
61
|
+
area_per_pixel_map = yg.read_raster('area_per_pixel.tif')
|
|
62
62
|
|
|
63
63
|
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
64
64
|
refined_elevation = (elevation_map >= species_min) && (elevation_map <= species_max)
|
|
@@ -71,8 +71,8 @@ print(f'area for species 123: {aoh.sum()}')
|
|
|
71
71
|
Similarly, you could save the result to a new raster layer:
|
|
72
72
|
|
|
73
73
|
```python
|
|
74
|
-
|
|
75
|
-
aoh.
|
|
74
|
+
...
|
|
75
|
+
aoh.to_geotiff("result.tif")
|
|
76
76
|
```
|
|
77
77
|
|
|
78
78
|
Yirgacheffe will automatically infer if you want to do an intersection of maps or a union of the maps based on the operators you use (see below for a full table). You can explicitly override that if you want.
|
|
@@ -107,30 +107,31 @@ If you have set either the intersection window or union window on a layer and yo
|
|
|
107
107
|
|
|
108
108
|
### Direct access to data
|
|
109
109
|
|
|
110
|
-
If doing per-layer operations isn't applicable for your application, you can read the pixel values for all layers (including VectorLayers) by calling `read_array` similarly to how you would for
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
### Todo but not supported
|
|
114
|
-
|
|
115
|
-
Yirgacheffe is work in progress, so things planned but not supported currently:
|
|
116
|
-
|
|
117
|
-
* Dynamic pixel scale adjustment - all raster layers must be provided at the same pixel scale currently *NOW IN EXPERIMENTAL TESTING, SEE BELOW*
|
|
118
|
-
* A fold operation
|
|
119
|
-
* CUDA/Metal support via CUPY/MLX
|
|
120
|
-
* Dispatching work across multiple CPUs *NOW IN EXPERIMENTAL TESTING, SEE BELOW*
|
|
121
|
-
|
|
110
|
+
If doing per-layer operations isn't applicable for your application, you can read the pixel values for all layers (including VectorLayers) by calling `read_array` similarly to how you would for GDAL. The data you access will be relative to the specified window - that is, if you've called either `set_window_for_intersection` or `set_window_for_union` then `read_array` will be relative to that and Yirgacheffe will clip or expand the data with zero values as necessary.
|
|
122
111
|
|
|
123
112
|
|
|
124
113
|
## Layer types
|
|
125
114
|
|
|
115
|
+
Note that as part of the move to the next major release, 2.0, we are adding simpler ways to create layers. Not all of those have been implemented yet, which is why this section has some inconsistencies. However, given many of the common cases are already covered, we present the new 2.0 style methods (`read_raster` and similar) here so you can write cleaner code today rather than making people wait for the final 2.0 release.
|
|
116
|
+
|
|
126
117
|
### RasterLayer
|
|
127
118
|
|
|
128
119
|
This is your basic GDAL raster layer, which you load from a geotiff.
|
|
129
120
|
|
|
130
121
|
```python
|
|
122
|
+
from yirgaceffe.layers import RasterLayer
|
|
123
|
+
|
|
131
124
|
with RasterLayer.layer_from_file('test1.tif') as layer:
|
|
132
|
-
|
|
133
|
-
|
|
125
|
+
total = layer.sum()
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
The new 2.0 way of doing this is:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
import yirgacheffe as yg
|
|
132
|
+
|
|
133
|
+
with yg.read_raster('test.tif') as layer:
|
|
134
|
+
total = layer.sum()
|
|
134
135
|
```
|
|
135
136
|
|
|
136
137
|
You can also create empty layers ready for you to store results, either by taking the dimensions from an existing layer. In both these cases you can either provide a filename to which the data will be written, or if you do not provide a filename then the layer will only exist in memory - this will be more efficient if the layer is being used for intermediary results.
|
|
@@ -159,6 +160,7 @@ with RasterLayer.layer_from_file('test1.tif') as source:
|
|
|
159
160
|
scaled = RasterLayer.scaled_raster_from_raster(source, PixelScale(0.0001, -0.0001), 'scaled.tif')
|
|
160
161
|
```
|
|
161
162
|
|
|
163
|
+
If the data is from a GeoTIFF that has a nodata value specified, then pixel values with that specified nodata value in them will be converted to NaN. You can override that by providing `ignore_nodata=True` as an optional argument to `layer_from_file` (or with the new 2.0 API, `read_raster`). You can find out if a layer has a nodata value by accessing the `nodata` property - it is None if there is no such value.
|
|
162
164
|
|
|
163
165
|
### VectorLayer
|
|
164
166
|
|
|
@@ -167,51 +169,30 @@ This layer will load vector data and rasterize it on demand as part of a calcula
|
|
|
167
169
|
Because it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
168
170
|
|
|
169
171
|
```python
|
|
170
|
-
|
|
172
|
+
from yirgaceffe import WGS_84_PROJECTION
|
|
173
|
+
from yirgaceffe.window import PixelScale
|
|
174
|
+
from yirgaceffe.layers import VectorLayer
|
|
175
|
+
|
|
176
|
+
with VectorLayer.layer_from_file('range.gpkg', PixelScale(0.001, -0.001), WGS_84_PROJECTION) as layer:
|
|
171
177
|
...
|
|
172
178
|
```
|
|
173
179
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
### UniformAreaLayer
|
|
178
|
-
|
|
179
|
-
In certain calculations you find you have a layer where all the rows of data are the same - notably geotiffs that contain the area of a given pixel do this due to how conventional map projections work. It's hugely inefficient to load the full map into memory, so whilst you could just load them as `Layer` types, we recommend you do:
|
|
180
|
+
The new 2.0 way of doing this is:
|
|
180
181
|
|
|
181
182
|
```python
|
|
182
|
-
|
|
183
|
-
....
|
|
184
|
-
```
|
|
185
|
-
|
|
186
|
-
Note that loading this data can still be very slow, due to how image compression works. So if you plan to use area.tiff more than once, we recommend use save an optimised version - this will do the slow uncompression once and then save a minimal file to speed up future processing:
|
|
183
|
+
import yirgacheffe as yg
|
|
187
184
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
UniformAreaLayer.generate_narrow_area_projection('area.tiff', 'yirgacheffe_area.tiff')
|
|
191
|
-
area_layer = UniformAreaLayer('yirgacheffe_area.tiff')
|
|
185
|
+
with yg.read_shape('range.gpkg', (0.001, -0.001), WGS_84_PROJECTION) as layer:
|
|
186
|
+
...
|
|
192
187
|
```
|
|
193
188
|
|
|
189
|
+
It is more common that when a shape file is loaded that its pixel size and projection will want to be made to match that of an existing raster (as per the opening area of habitat example). For that there is the following convenience method:
|
|
194
190
|
|
|
195
|
-
### ConstantLayer
|
|
196
|
-
|
|
197
|
-
This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
|
|
198
191
|
|
|
199
192
|
```python
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
area_layer = ConstantLayer(0.0)
|
|
204
|
-
```
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
### H3CellLayer
|
|
208
|
-
|
|
209
|
-
If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.
|
|
210
|
-
|
|
211
|
-
Becuase it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
212
|
-
|
|
213
|
-
```python
|
|
214
|
-
hex_cell_layer = H3CellLayer('88972eac11fffff', layer1.pixel_scale, layer1.projection)
|
|
193
|
+
with yg.read_raster("test.tif") as raster_layer:
|
|
194
|
+
with yg.read_shape_like('range.gpkg', raster_layer) as shape_layer:
|
|
195
|
+
...
|
|
215
196
|
```
|
|
216
197
|
|
|
217
198
|
### GroupLayer
|
|
@@ -240,6 +221,18 @@ with GroupLayer.layer_from_directory('.') as all_tiles:
|
|
|
240
221
|
...
|
|
241
222
|
```
|
|
242
223
|
|
|
224
|
+
The new 2.0 way of doing this is:
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
import yirgacheffe as yg
|
|
228
|
+
|
|
229
|
+
with yg.read_rasters(['tile_N10_E10.tif', 'tile_N20_E10.tif']) as all_tiles:
|
|
230
|
+
...
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
If any of the layers have a `nodata` value specified, then any pixel with that value will be masked out to allow data from other layers to be visible.
|
|
234
|
+
|
|
235
|
+
|
|
243
236
|
### TiledGroupLayer
|
|
244
237
|
|
|
245
238
|
This is a specialisation of GroupLayer, which you can use if your layers are all the same size and form a grid, as is often the case with map tiles. In this case the rendering code can be optimised and this class is significantly faster that GroupLayer.
|
|
@@ -250,11 +243,59 @@ tile2 = RasterLayer.layer_from_file('tile_N20_E10.tif')
|
|
|
250
243
|
all_tiles = TiledGroupLayer([tile1, tile2])
|
|
251
244
|
```
|
|
252
245
|
|
|
246
|
+
The new 2.0 way of doing this is:
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
import yirgacheffe as yg
|
|
250
|
+
|
|
251
|
+
with yg.read_rasters(['tile_N10_E10.tif', 'tile_N20_E10.tif'], tiled=True) as all_tiles:
|
|
252
|
+
...
|
|
253
|
+
```
|
|
254
|
+
|
|
253
255
|
Notes:
|
|
254
256
|
|
|
255
257
|
* You can have missing tiles, and these will be filled in with zeros.
|
|
256
258
|
* You can have tiles that overlap, so long as they still conform to the rule that all tiles are the same size and on a grid.
|
|
257
259
|
|
|
260
|
+
### ConstantLayer
|
|
261
|
+
|
|
262
|
+
This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
|
|
263
|
+
|
|
264
|
+
```python
|
|
265
|
+
try:
|
|
266
|
+
area_layer = RasterLayer.layer_from_file('myarea.tiff')
|
|
267
|
+
except FileDoesNotExist:
|
|
268
|
+
area_layer = ConstantLayer(0.0)
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
### H3CellLayer
|
|
272
|
+
|
|
273
|
+
If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.
|
|
274
|
+
|
|
275
|
+
Becuase it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
276
|
+
|
|
277
|
+
```python
|
|
278
|
+
hex_cell_layer = H3CellLayer('88972eac11fffff', layer1.pixel_scale, layer1.projection)
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
### UniformAreaLayer
|
|
283
|
+
|
|
284
|
+
In certain calculations you find you have a layer where all the rows of data are the same - notably geotiffs that contain the area of a given pixel do this due to how conventional map projections work. It's hugely inefficient to load the full map into memory, so whilst you could just load them as `Layer` types, we recommend you do:
|
|
285
|
+
|
|
286
|
+
```python
|
|
287
|
+
with UniformAreaLayer('area.tiff') as layer:
|
|
288
|
+
....
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Note that loading this data can still be very slow, due to how image compression works. So if you plan to use area.tiff more than once, we recommend use save an optimised version - this will do the slow uncompression once and then save a minimal file to speed up future processing:
|
|
292
|
+
|
|
293
|
+
```python
|
|
294
|
+
if not os.path.exists('yirgacheffe_area.tiff'):
|
|
295
|
+
UniformAreaLayer.generate_narrow_area_projection('area.tiff', 'yirgacheffe_area.tiff')
|
|
296
|
+
area_layer = UniformAreaLayer('yirgacheffe_area.tiff')
|
|
297
|
+
```
|
|
298
|
+
|
|
258
299
|
|
|
259
300
|
## Supported operations on layers
|
|
260
301
|
|
|
@@ -281,6 +322,24 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
|
|
|
281
322
|
calc.save(result)
|
|
282
323
|
```
|
|
283
324
|
|
|
325
|
+
|
|
326
|
+
The new 2.0 way of doing these are:
|
|
327
|
+
|
|
328
|
+
```python
|
|
329
|
+
with yg.read_raster('test1.tif') as layer1:
|
|
330
|
+
with yg.read_raster('test2.tif') as layer2:
|
|
331
|
+
result = layer1 + layer2
|
|
332
|
+
result.to_geotiff("result.tif")
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
or
|
|
336
|
+
|
|
337
|
+
```python
|
|
338
|
+
with yg.read_raster('test1.tif') as layer1:
|
|
339
|
+
result = layer1 * 42.0
|
|
340
|
+
result.to_geotiff("result.tif")
|
|
341
|
+
```
|
|
342
|
+
|
|
284
343
|
### Boolean testing
|
|
285
344
|
|
|
286
345
|
Testing for equality, less than, less than or equal, greater than, and greater than or equal are supported on layers, along with logical or and logical and, as per this example, where `elevation_upper` and `elevation_lower` are scalar values:
|
|
@@ -28,12 +28,12 @@ The motivation for Yirgacheffe layers is to make working with gdal data slightly
|
|
|
28
28
|
For example, if we wanted to do a simple [Area of Habitat](https://github.com/quantifyearth/aoh-calculator/) calculation, whereby we find the pixels where a species resides by combining its range polygon, its habitat preferences, and its elevation preferences, the code would be like this:
|
|
29
29
|
|
|
30
30
|
```python
|
|
31
|
-
|
|
31
|
+
import yirgaceffe as yg
|
|
32
32
|
|
|
33
|
-
habitat_map =
|
|
34
|
-
elevation_map =
|
|
35
|
-
range_polygon =
|
|
36
|
-
area_per_pixel_map =
|
|
33
|
+
habitat_map = yg.read_raster("habitats.tif")
|
|
34
|
+
elevation_map = yg.read_raster('elevation.tif')
|
|
35
|
+
range_polygon = yg.read_shape_like('species123.geojson', like=habitat_map)
|
|
36
|
+
area_per_pixel_map = yg.read_raster('area_per_pixel.tif')
|
|
37
37
|
|
|
38
38
|
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
39
39
|
refined_elevation = (elevation_map >= species_min) && (elevation_map <= species_max)
|
|
@@ -46,8 +46,8 @@ print(f'area for species 123: {aoh.sum()}')
|
|
|
46
46
|
Similarly, you could save the result to a new raster layer:
|
|
47
47
|
|
|
48
48
|
```python
|
|
49
|
-
|
|
50
|
-
aoh.
|
|
49
|
+
...
|
|
50
|
+
aoh.to_geotiff("result.tif")
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
Yirgacheffe will automatically infer if you want to do an intersection of maps or a union of the maps based on the operators you use (see below for a full table). You can explicitly override that if you want.
|
|
@@ -82,30 +82,31 @@ If you have set either the intersection window or union window on a layer and yo
|
|
|
82
82
|
|
|
83
83
|
### Direct access to data
|
|
84
84
|
|
|
85
|
-
If doing per-layer operations isn't applicable for your application, you can read the pixel values for all layers (including VectorLayers) by calling `read_array` similarly to how you would for
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
### Todo but not supported
|
|
89
|
-
|
|
90
|
-
Yirgacheffe is work in progress, so things planned but not supported currently:
|
|
91
|
-
|
|
92
|
-
* Dynamic pixel scale adjustment - all raster layers must be provided at the same pixel scale currently *NOW IN EXPERIMENTAL TESTING, SEE BELOW*
|
|
93
|
-
* A fold operation
|
|
94
|
-
* CUDA/Metal support via CUPY/MLX
|
|
95
|
-
* Dispatching work across multiple CPUs *NOW IN EXPERIMENTAL TESTING, SEE BELOW*
|
|
96
|
-
|
|
85
|
+
If doing per-layer operations isn't applicable for your application, you can read the pixel values for all layers (including VectorLayers) by calling `read_array` similarly to how you would for GDAL. The data you access will be relative to the specified window - that is, if you've called either `set_window_for_intersection` or `set_window_for_union` then `read_array` will be relative to that and Yirgacheffe will clip or expand the data with zero values as necessary.
|
|
97
86
|
|
|
98
87
|
|
|
99
88
|
## Layer types
|
|
100
89
|
|
|
90
|
+
Note that as part of the move to the next major release, 2.0, we are adding simpler ways to create layers. Not all of those have been implemented yet, which is why this section has some inconsistencies. However, given many of the common cases are already covered, we present the new 2.0 style methods (`read_raster` and similar) here so you can write cleaner code today rather than making people wait for the final 2.0 release.
|
|
91
|
+
|
|
101
92
|
### RasterLayer
|
|
102
93
|
|
|
103
94
|
This is your basic GDAL raster layer, which you load from a geotiff.
|
|
104
95
|
|
|
105
96
|
```python
|
|
97
|
+
from yirgaceffe.layers import RasterLayer
|
|
98
|
+
|
|
106
99
|
with RasterLayer.layer_from_file('test1.tif') as layer:
|
|
107
|
-
|
|
108
|
-
|
|
100
|
+
total = layer.sum()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The new 2.0 way of doing this is:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
import yirgacheffe as yg
|
|
107
|
+
|
|
108
|
+
with yg.read_raster('test.tif') as layer:
|
|
109
|
+
total = layer.sum()
|
|
109
110
|
```
|
|
110
111
|
|
|
111
112
|
You can also create empty layers ready for you to store results, either by taking the dimensions from an existing layer. In both these cases you can either provide a filename to which the data will be written, or if you do not provide a filename then the layer will only exist in memory - this will be more efficient if the layer is being used for intermediary results.
|
|
@@ -134,6 +135,7 @@ with RasterLayer.layer_from_file('test1.tif') as source:
|
|
|
134
135
|
scaled = RasterLayer.scaled_raster_from_raster(source, PixelScale(0.0001, -0.0001), 'scaled.tif')
|
|
135
136
|
```
|
|
136
137
|
|
|
138
|
+
If the data is from a GeoTIFF that has a nodata value specified, then pixel values with that specified nodata value in them will be converted to NaN. You can override that by providing `ignore_nodata=True` as an optional argument to `layer_from_file` (or with the new 2.0 API, `read_raster`). You can find out if a layer has a nodata value by accessing the `nodata` property - it is None if there is no such value.
|
|
137
139
|
|
|
138
140
|
### VectorLayer
|
|
139
141
|
|
|
@@ -142,51 +144,30 @@ This layer will load vector data and rasterize it on demand as part of a calcula
|
|
|
142
144
|
Because it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
143
145
|
|
|
144
146
|
```python
|
|
145
|
-
|
|
147
|
+
from yirgaceffe import WGS_84_PROJECTION
|
|
148
|
+
from yirgaceffe.window import PixelScale
|
|
149
|
+
from yirgaceffe.layers import VectorLayer
|
|
150
|
+
|
|
151
|
+
with VectorLayer.layer_from_file('range.gpkg', PixelScale(0.001, -0.001), WGS_84_PROJECTION) as layer:
|
|
146
152
|
...
|
|
147
153
|
```
|
|
148
154
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
### UniformAreaLayer
|
|
153
|
-
|
|
154
|
-
In certain calculations you find you have a layer where all the rows of data are the same - notably geotiffs that contain the area of a given pixel do this due to how conventional map projections work. It's hugely inefficient to load the full map into memory, so whilst you could just load them as `Layer` types, we recommend you do:
|
|
155
|
+
The new 2.0 way of doing this is:
|
|
155
156
|
|
|
156
157
|
```python
|
|
157
|
-
|
|
158
|
-
....
|
|
159
|
-
```
|
|
160
|
-
|
|
161
|
-
Note that loading this data can still be very slow, due to how image compression works. So if you plan to use area.tiff more than once, we recommend use save an optimised version - this will do the slow uncompression once and then save a minimal file to speed up future processing:
|
|
158
|
+
import yirgacheffe as yg
|
|
162
159
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
UniformAreaLayer.generate_narrow_area_projection('area.tiff', 'yirgacheffe_area.tiff')
|
|
166
|
-
area_layer = UniformAreaLayer('yirgacheffe_area.tiff')
|
|
160
|
+
with yg.read_shape('range.gpkg', (0.001, -0.001), WGS_84_PROJECTION) as layer:
|
|
161
|
+
...
|
|
167
162
|
```
|
|
168
163
|
|
|
164
|
+
It is more common that when a shape file is loaded that its pixel size and projection will want to be made to match that of an existing raster (as per the opening area of habitat example). For that there is the following convenience method:
|
|
169
165
|
|
|
170
|
-
### ConstantLayer
|
|
171
|
-
|
|
172
|
-
This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
|
|
173
166
|
|
|
174
167
|
```python
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
area_layer = ConstantLayer(0.0)
|
|
179
|
-
```
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
### H3CellLayer
|
|
183
|
-
|
|
184
|
-
If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.
|
|
185
|
-
|
|
186
|
-
Becuase it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
187
|
-
|
|
188
|
-
```python
|
|
189
|
-
hex_cell_layer = H3CellLayer('88972eac11fffff', layer1.pixel_scale, layer1.projection)
|
|
168
|
+
with yg.read_raster("test.tif") as raster_layer:
|
|
169
|
+
with yg.read_shape_like('range.gpkg', raster_layer) as shape_layer:
|
|
170
|
+
...
|
|
190
171
|
```
|
|
191
172
|
|
|
192
173
|
### GroupLayer
|
|
@@ -215,6 +196,18 @@ with GroupLayer.layer_from_directory('.') as all_tiles:
|
|
|
215
196
|
...
|
|
216
197
|
```
|
|
217
198
|
|
|
199
|
+
The new 2.0 way of doing this is:
|
|
200
|
+
|
|
201
|
+
```python
|
|
202
|
+
import yirgacheffe as yg
|
|
203
|
+
|
|
204
|
+
with yg.read_rasters(['tile_N10_E10.tif', 'tile_N20_E10.tif']) as all_tiles:
|
|
205
|
+
...
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
If any of the layers have a `nodata` value specified, then any pixel with that value will be masked out to allow data from other layers to be visible.
|
|
209
|
+
|
|
210
|
+
|
|
218
211
|
### TiledGroupLayer
|
|
219
212
|
|
|
220
213
|
This is a specialisation of GroupLayer, which you can use if your layers are all the same size and form a grid, as is often the case with map tiles. In this case the rendering code can be optimised and this class is significantly faster that GroupLayer.
|
|
@@ -225,11 +218,59 @@ tile2 = RasterLayer.layer_from_file('tile_N20_E10.tif')
|
|
|
225
218
|
all_tiles = TiledGroupLayer([tile1, tile2])
|
|
226
219
|
```
|
|
227
220
|
|
|
221
|
+
The new 2.0 way of doing this is:
|
|
222
|
+
|
|
223
|
+
```python
|
|
224
|
+
import yirgacheffe as yg
|
|
225
|
+
|
|
226
|
+
with yg.read_rasters(['tile_N10_E10.tif', 'tile_N20_E10.tif'], tiled=True) as all_tiles:
|
|
227
|
+
...
|
|
228
|
+
```
|
|
229
|
+
|
|
228
230
|
Notes:
|
|
229
231
|
|
|
230
232
|
* You can have missing tiles, and these will be filled in with zeros.
|
|
231
233
|
* You can have tiles that overlap, so long as they still conform to the rule that all tiles are the same size and on a grid.
|
|
232
234
|
|
|
235
|
+
### ConstantLayer
|
|
236
|
+
|
|
237
|
+
This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
try:
|
|
241
|
+
area_layer = RasterLayer.layer_from_file('myarea.tiff')
|
|
242
|
+
except FileDoesNotExist:
|
|
243
|
+
area_layer = ConstantLayer(0.0)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
### H3CellLayer
|
|
247
|
+
|
|
248
|
+
If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.
|
|
249
|
+
|
|
250
|
+
Becuase it will be rasterized you need to specify the pixel scale and map projection to be used when rasterising the data, and the common way to do that is by using one of your other layers.
|
|
251
|
+
|
|
252
|
+
```python
|
|
253
|
+
hex_cell_layer = H3CellLayer('88972eac11fffff', layer1.pixel_scale, layer1.projection)
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
### UniformAreaLayer
|
|
258
|
+
|
|
259
|
+
In certain calculations you find you have a layer where all the rows of data are the same - notably geotiffs that contain the area of a given pixel do this due to how conventional map projections work. It's hugely inefficient to load the full map into memory, so whilst you could just load them as `Layer` types, we recommend you do:
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
with UniformAreaLayer('area.tiff') as layer:
|
|
263
|
+
....
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Note that loading this data can still be very slow, due to how image compression works. So if you plan to use area.tiff more than once, we recommend use save an optimised version - this will do the slow uncompression once and then save a minimal file to speed up future processing:
|
|
267
|
+
|
|
268
|
+
```python
|
|
269
|
+
if not os.path.exists('yirgacheffe_area.tiff'):
|
|
270
|
+
UniformAreaLayer.generate_narrow_area_projection('area.tiff', 'yirgacheffe_area.tiff')
|
|
271
|
+
area_layer = UniformAreaLayer('yirgacheffe_area.tiff')
|
|
272
|
+
```
|
|
273
|
+
|
|
233
274
|
|
|
234
275
|
## Supported operations on layers
|
|
235
276
|
|
|
@@ -256,6 +297,24 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
|
|
|
256
297
|
calc.save(result)
|
|
257
298
|
```
|
|
258
299
|
|
|
300
|
+
|
|
301
|
+
The new 2.0 way of doing these are:
|
|
302
|
+
|
|
303
|
+
```python
|
|
304
|
+
with yg.read_raster('test1.tif') as layer1:
|
|
305
|
+
with yg.read_raster('test2.tif') as layer2:
|
|
306
|
+
result = layer1 + layer2
|
|
307
|
+
result.to_geotiff("result.tif")
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
or
|
|
311
|
+
|
|
312
|
+
```python
|
|
313
|
+
with yg.read_raster('test1.tif') as layer1:
|
|
314
|
+
result = layer1 * 42.0
|
|
315
|
+
result.to_geotiff("result.tif")
|
|
316
|
+
```
|
|
317
|
+
|
|
259
318
|
### Boolean testing
|
|
260
319
|
|
|
261
320
|
Testing for equality, less than, less than or equal, greater than, and greater than or equal are supported on layers, along with logical or and logical and, as per this example, where `elevation_upper` and `elevation_lower` are scalar values:
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "yirgacheffe"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.6.0"
|
|
10
10
|
description = "Abstraction of gdal datasets for doing basic math operations"
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]
|
|
@@ -19,3 +19,13 @@ def test_area_operators(lhs: Area, rhs: Area, is_equal: bool, overlaps: bool) ->
|
|
|
19
19
|
assert (lhs != rhs) == (not is_equal)
|
|
20
20
|
assert (lhs.overlaps(rhs)) == overlaps
|
|
21
21
|
assert (rhs.overlaps(lhs)) == overlaps
|
|
22
|
+
assert not lhs.is_world
|
|
23
|
+
assert not rhs.is_world
|
|
24
|
+
|
|
25
|
+
def test_global_area() -> None:
|
|
26
|
+
area = Area.world()
|
|
27
|
+
assert area.is_world
|
|
28
|
+
|
|
29
|
+
other_area = Area(-10.0, 10.0, 10.0, -10.0)
|
|
30
|
+
assert area.overlaps(other_area)
|
|
31
|
+
assert other_area.overlaps(area)
|
|
@@ -44,7 +44,6 @@ def test_round_trip_from_gdal(ytype) -> None:
|
|
|
44
44
|
def test_round_trip_float64() -> None:
|
|
45
45
|
backend_type = backend.dtype_to_backed(DataType.Float64)
|
|
46
46
|
ytype = backend.backend_to_dtype(backend_type)
|
|
47
|
-
print(BACKEND, "sad")
|
|
48
47
|
match BACKEND:
|
|
49
48
|
case "NUMPY":
|
|
50
49
|
assert ytype == DataType.Float64
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from yirgacheffe.layers.rasters import RasterLayer
|
|
4
|
+
from yirgacheffe.layers.group import GroupLayer
|
|
5
|
+
|
|
6
|
+
from tests.helpers import gdal_dataset_with_data
|
|
7
|
+
|
|
8
|
+
def test_raster_without_nodata_value() -> None:
|
|
9
|
+
data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 5.0, 8.0]])
|
|
10
|
+
dataset = gdal_dataset_with_data((0.0, 0.0), 0.02, data1)
|
|
11
|
+
with RasterLayer(dataset) as layer:
|
|
12
|
+
assert layer.nodata is None
|
|
13
|
+
actual = layer.read_array(0, 0, 4, 2)
|
|
14
|
+
assert np.array_equal(data1, actual, equal_nan=True)
|
|
15
|
+
|
|
16
|
+
def test_raster_with_nodata_value() -> None:
|
|
17
|
+
data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 5.0, 8.0]])
|
|
18
|
+
dataset = gdal_dataset_with_data((0.0, 0.0), 0.02, data1)
|
|
19
|
+
dataset.GetRasterBand(1).SetNoDataValue(5.0)
|
|
20
|
+
with RasterLayer(dataset) as layer:
|
|
21
|
+
assert layer.nodata == 5.0
|
|
22
|
+
data1[data1 == 5.0] = np.nan
|
|
23
|
+
actual = layer.read_array(0, 0, 4, 2)
|
|
24
|
+
assert np.array_equal(data1, actual, equal_nan=True)
|
|
25
|
+
|
|
26
|
+
def test_raster_with_nodata_value_ignored() -> None:
|
|
27
|
+
data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 5.0, 8.0]])
|
|
28
|
+
dataset = gdal_dataset_with_data((0.0, 0.0), 0.02, data1)
|
|
29
|
+
dataset.GetRasterBand(1).SetNoDataValue(5.0)
|
|
30
|
+
with RasterLayer(dataset, ignore_nodata=True) as layer:
|
|
31
|
+
assert layer.nodata == 5.0
|
|
32
|
+
actual = layer.read_array(0, 0, 4, 2)
|
|
33
|
+
assert np.array_equal(data1, actual, equal_nan=True)
|
|
34
|
+
|
|
35
|
+
def test_group_layer_with_nodata_values() -> None:
|
|
36
|
+
data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 5.0, 5.0, 5.0]])
|
|
37
|
+
dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1)
|
|
38
|
+
dataset1.GetRasterBand(1).SetNoDataValue(5.0)
|
|
39
|
+
|
|
40
|
+
data2 = np.array([[1.0, 1.0, 1.0, 1.0], [5.0, 6.0, 7.0, 8.0]])
|
|
41
|
+
dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2)
|
|
42
|
+
dataset2.GetRasterBand(1).SetNoDataValue(1.0)
|
|
43
|
+
|
|
44
|
+
with RasterLayer(dataset1) as layer1:
|
|
45
|
+
with RasterLayer(dataset2) as layer2:
|
|
46
|
+
with GroupLayer([layer1, layer2]) as group:
|
|
47
|
+
actual = group.read_array(0, 0, 4, 2)
|
|
48
|
+
expected = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0]])
|
|
49
|
+
assert np.array_equal(expected, actual, equal_nan=True)
|
|
50
|
+
|
|
51
|
+
def test_group_layer_with_nodata_values_ignore_nodata() -> None:
|
|
52
|
+
data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.0, 5.0, 5.0, 5.0]])
|
|
53
|
+
dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1)
|
|
54
|
+
dataset1.GetRasterBand(1).SetNoDataValue(5.0)
|
|
55
|
+
|
|
56
|
+
data2 = np.array([[1.0, 1.0, 1.0, 1.0], [5.0, 6.0, 7.0, 8.0]])
|
|
57
|
+
dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2)
|
|
58
|
+
dataset2.GetRasterBand(1).SetNoDataValue(1.0)
|
|
59
|
+
|
|
60
|
+
with RasterLayer(dataset1, ignore_nodata=True) as layer1:
|
|
61
|
+
with RasterLayer(dataset2, ignore_nodata=True) as layer2:
|
|
62
|
+
with GroupLayer([layer1, layer2]) as group:
|
|
63
|
+
actual = group.read_array(0, 0, 4, 2)
|
|
64
|
+
assert np.array_equal(data1, actual, equal_nan=True)
|