yirgacheffe 1.5.0__tar.gz → 1.10.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yirgacheffe-1.10.2/PKG-INFO +150 -0
- yirgacheffe-1.10.2/README.md +102 -0
- yirgacheffe-1.10.2/pyproject.toml +83 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_area.py +3 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_auto_windowing.py +84 -20
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_constants.py +21 -2
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_datatypes.py +66 -4
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_h3layer.py +33 -19
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_intersection.py +71 -8
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_multiband.py +8 -6
- yirgacheffe-1.10.2/tests/test_nodata.py +92 -0
- yirgacheffe-1.10.2/tests/test_openers.py +292 -0
- yirgacheffe-1.10.2/tests/test_operator_hashing.py +372 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_operators.py +406 -63
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_optimisation.py +26 -24
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_parallel_operators.py +38 -5
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_pickle.py +8 -10
- yirgacheffe-1.10.2/tests/test_pixel_coord.py +257 -0
- yirgacheffe-1.10.2/tests/test_projection.py +45 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_raster.py +57 -10
- yirgacheffe-1.10.2/tests/test_reduce.py +24 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_rescaling.py +28 -21
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_rounding.py +1 -2
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_union.py +50 -6
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_vectors.py +17 -12
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_window.py +10 -0
- yirgacheffe-1.10.2/yirgacheffe/__init__.py +25 -0
- yirgacheffe-1.10.2/yirgacheffe/_backends/enumeration.py +150 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/_backends/mlx.py +15 -4
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/_backends/numpy.py +19 -4
- yirgacheffe-1.10.2/yirgacheffe/_core.py +212 -0
- yirgacheffe-1.10.2/yirgacheffe/_operators/__init__.py +1393 -0
- yirgacheffe-1.10.2/yirgacheffe/_operators/cse.py +66 -0
- yirgacheffe-1.10.2/yirgacheffe/constants.py +8 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/area.py +19 -9
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/base.py +155 -95
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/constant.py +28 -9
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/group.py +65 -34
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/h3layer.py +39 -21
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/rasters.py +85 -40
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/rescaled.py +35 -14
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/vectors.py +184 -74
- yirgacheffe-1.10.2/yirgacheffe/operators.py +7 -0
- yirgacheffe-1.10.2/yirgacheffe/py.typed +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/rounding.py +5 -3
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/window.py +105 -84
- yirgacheffe-1.10.2/yirgacheffe.egg-info/PKG-INFO +150 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe.egg-info/SOURCES.txt +8 -1
- yirgacheffe-1.10.2/yirgacheffe.egg-info/requires.txt +28 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe.egg-info/top_level.txt +1 -0
- yirgacheffe-1.5.0/PKG-INFO +0 -553
- yirgacheffe-1.5.0/README.md +0 -528
- yirgacheffe-1.5.0/pyproject.toml +0 -39
- yirgacheffe-1.5.0/tests/test_base.py +0 -196
- yirgacheffe-1.5.0/tests/test_openers.py +0 -165
- yirgacheffe-1.5.0/yirgacheffe/__init__.py +0 -11
- yirgacheffe-1.5.0/yirgacheffe/_backends/enumeration.py +0 -60
- yirgacheffe-1.5.0/yirgacheffe/_core.py +0 -133
- yirgacheffe-1.5.0/yirgacheffe/constants.py +0 -8
- yirgacheffe-1.5.0/yirgacheffe/operators.py +0 -877
- yirgacheffe-1.5.0/yirgacheffe.egg-info/PKG-INFO +0 -553
- yirgacheffe-1.5.0/yirgacheffe.egg-info/requires.txt +0 -13
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/LICENSE +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/MANIFEST.in +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/setup.cfg +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_group.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_save_with_window.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_sum_with_window.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/tests/test_uniform_area_layer.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/_backends/__init__.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe/layers/__init__.py +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe.egg-info/dependency_links.txt +0 -0
- {yirgacheffe-1.5.0 → yirgacheffe-1.10.2}/yirgacheffe.egg-info/entry_points.txt +0 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: yirgacheffe
|
|
3
|
+
Version: 1.10.2
|
|
4
|
+
Summary: Abstraction of gdal datasets for doing basic math operations
|
|
5
|
+
Author-email: Michael Dales <mwd24@cam.ac.uk>
|
|
6
|
+
License-Expression: ISC
|
|
7
|
+
Project-URL: Homepage, https://yirgacheffe.org/
|
|
8
|
+
Project-URL: Repository, https://github.com/quantifyearth/yirgacheffe.git
|
|
9
|
+
Project-URL: Issues, https://github.com/quantifyearth/yirgacheffe/issues
|
|
10
|
+
Project-URL: Changelog, https://yirgacheffe.org/latest/changelog/
|
|
11
|
+
Keywords: gdal,gis,geospatial,declarative
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy<3.0,>=1.24
|
|
23
|
+
Requires-Dist: gdal[numpy]<4.0,>=3.8
|
|
24
|
+
Requires-Dist: scikit-image<1.0,>=0.20
|
|
25
|
+
Requires-Dist: torch
|
|
26
|
+
Requires-Dist: dill
|
|
27
|
+
Requires-Dist: deprecation
|
|
28
|
+
Requires-Dist: tomli
|
|
29
|
+
Requires-Dist: h3
|
|
30
|
+
Requires-Dist: pyproj
|
|
31
|
+
Provides-Extra: mlx
|
|
32
|
+
Requires-Dist: mlx; extra == "mlx"
|
|
33
|
+
Provides-Extra: matplotlib
|
|
34
|
+
Requires-Dist: matplotlib; extra == "matplotlib"
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: mypy; extra == "dev"
|
|
37
|
+
Requires-Dist: pylint; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest; extra == "dev"
|
|
39
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
40
|
+
Requires-Dist: pytest-mock; extra == "dev"
|
|
41
|
+
Requires-Dist: build; extra == "dev"
|
|
42
|
+
Requires-Dist: twine; extra == "dev"
|
|
43
|
+
Requires-Dist: mkdocs-material; extra == "dev"
|
|
44
|
+
Requires-Dist: mkdocstrings-python; extra == "dev"
|
|
45
|
+
Requires-Dist: mike; extra == "dev"
|
|
46
|
+
Requires-Dist: mkdocs-gen-files; extra == "dev"
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
|
|
49
|
+
# Yirgacheffe: a declarative geospatial library for Python to make data-science with maps easier
|
|
50
|
+
|
|
51
|
+
[](https://github.com/quantifyearth/yirgacheffe/actions)
|
|
52
|
+
[](https://yirgacheffe.org)
|
|
53
|
+
[](https://pypi.org/project/yirgacheffe/)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## Overview
|
|
57
|
+
|
|
58
|
+
Yirgacheffe is a declarative geospatial library, allowing you to operate on both raster and polygon geospatial datasets without having to do all the tedious book keeping around layer alignment or dealing with hardware concerns around memory or parallelism. you can load into memory safely.
|
|
59
|
+
|
|
60
|
+
Example common use-cases:
|
|
61
|
+
|
|
62
|
+
* Do the datasets overlap? Yirgacheffe will let you define either the intersection or the union of a set of different datasets, scaling up or down the area as required.
|
|
63
|
+
* Rasterisation of vector layers: if you have a vector dataset then you can add that to your computation and yirgaceffe will rasterize it on demand, so you never need to store more data in memory than necessary.
|
|
64
|
+
* Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
|
|
65
|
+
* Parallelisation of operations over many CPU cores.
|
|
66
|
+
* Built in support for optionally using GPUs via [MLX](https://ml-explore.github.io/mlx/build/html/index.html) support.
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
Yirgacheffe is available via pypi, so can be installed with pip for example:
|
|
71
|
+
|
|
72
|
+
```SystemShell
|
|
73
|
+
$ pip install yirgacheffe
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Documentation
|
|
77
|
+
|
|
78
|
+
The documentation can be found on [yirgacheffe.org](https://yirgacheffe.org/)
|
|
79
|
+
|
|
80
|
+
## Simple examples:
|
|
81
|
+
|
|
82
|
+
Here is how to do cloud removal from [Sentinel-2 data](https://browser.dataspace.copernicus.eu/?zoom=14&lat=6.15468&lng=38.20581&themeId=DEFAULT-THEME&visualizationUrl=U2FsdGVkX1944lrmeTJcaSsnoxNMp4oucN1AjklGUANHd2cRZWyXnepHvzpaOWzMhH8SrWQo%2BqrOvOnu6f9FeCMrS%2FDZmvjzID%2FoE1tbOCEHK8ohPXjFqYojeR9%2B82ri&datasetId=S2_L2A_CDAS&fromTime=2025-09-09T00%3A00%3A00.000Z&toTime=2025-09-09T23%3A59%3A59.999Z&layerId=1_TRUE_COLOR&demSource3D=%22MAPZEN%22&cloudCoverage=30&dateMode=SINGLE), using the [Scene Classification Layer](https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/scene-classification/) data:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import yirgaceffe as yg
|
|
86
|
+
|
|
87
|
+
with (
|
|
88
|
+
yg.read_raster("T37NCG_20250909T073609_B06_20m.jp2") as vre2,
|
|
89
|
+
yg.read_raster("T37NCG_20250909T073609_SCL_20m.jp2") as scl,
|
|
90
|
+
):
|
|
91
|
+
is_cloud = (scl == 8) | (scl == 9) | (scl == 10) # various cloud types
|
|
92
|
+
is_shadow = (scl == 3)
|
|
93
|
+
is_bad = is_cloud | is_shadow
|
|
94
|
+
|
|
95
|
+
masked_vre2 = yg.where(is_bad, float("nan"), vre2)
|
|
96
|
+
masked_vre2.to_geotiff("vre2_cleaned.tif")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
or a species' [Area of Habitat](https://www.sciencedirect.com/science/article/pii/S0169534719301892) calculation:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import yirgaceffe as yg
|
|
103
|
+
|
|
104
|
+
with (
|
|
105
|
+
yg.read_raster("habitats.tif") as habitat_map,
|
|
106
|
+
yg.read_raster('elevation.tif') as elevation_map,
|
|
107
|
+
yg.read_shape('species123.geojson') as range_map,
|
|
108
|
+
):
|
|
109
|
+
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
110
|
+
refined_elevation = (elevation_map >= species_min) & (elevation_map <= species_max)
|
|
111
|
+
aoh = refined_habitat * refined_elevation * range_polygon * area_per_pixel_map
|
|
112
|
+
print(f'Area of habitat: {aoh.sum()}')
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Citation
|
|
116
|
+
|
|
117
|
+
If you use Yirgacheffe in your research, please cite our paper:
|
|
118
|
+
|
|
119
|
+
> Michael Winston Dales, Alison Eyres, Patrick Ferris, Francesca A. Ridley, Simon Tarr, and Anil Madhavapeddy. 2025. Yirgacheffe: A Declarative Approach to Geospatial Data. In *Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet* (PROPL '25). Association for Computing Machinery, New York, NY, USA, 47–54. https://doi.org/10.1145/3759536.3763806
|
|
120
|
+
|
|
121
|
+
<details>
|
|
122
|
+
<summary>BibTeX</summary>
|
|
123
|
+
|
|
124
|
+
```bibtex
|
|
125
|
+
@inproceedings{10.1145/3759536.3763806,
|
|
126
|
+
author = {Dales, Michael Winston and Eyres, Alison and Ferris, Patrick and Ridley, Francesca A. and Tarr, Simon and Madhavapeddy, Anil},
|
|
127
|
+
title = {Yirgacheffe: A Declarative Approach to Geospatial Data},
|
|
128
|
+
year = {2025},
|
|
129
|
+
isbn = {9798400721618},
|
|
130
|
+
publisher = {Association for Computing Machinery},
|
|
131
|
+
address = {New York, NY, USA},
|
|
132
|
+
url = {https://doi.org/10.1145/3759536.3763806},
|
|
133
|
+
doi = {10.1145/3759536.3763806},
|
|
134
|
+
abstract = {We present Yirgacheffe, a declarative geospatial library that allows spatial algorithms to be implemented concisely, supports parallel execution, and avoids common errors by automatically handling data (large geospatial rasters) and resources (cores, memory, GPUs). Our primary user domain comprises ecologists, where a typical problem involves cleaning messy occurrence data, overlaying it over tiled rasters, combining layers, and deriving actionable insights from the results. We describe the successes of this approach towards driving key pipelines related to global biodiversity and describe the capability gaps that remain, hoping to motivate more research into geospatial domain-specific languages.},
|
|
135
|
+
booktitle = {Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet},
|
|
136
|
+
pages = {47–54},
|
|
137
|
+
numpages = {8},
|
|
138
|
+
keywords = {Biodiversity, Declarative, Geospatial, Python},
|
|
139
|
+
location = {Singapore, Singapore},
|
|
140
|
+
series = {PROPL '25}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
</details>
|
|
145
|
+
|
|
146
|
+
## Thanks
|
|
147
|
+
|
|
148
|
+
Thanks to discussion and feedback from my colleagues, particularly Alison Eyres, Patrick Ferris, Amelia Holcomb, and Anil Madhavapeddy.
|
|
149
|
+
|
|
150
|
+
Inspired by the work of Daniele Baisero in his AoH library.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Yirgacheffe: a declarative geospatial library for Python to make data-science with maps easier
|
|
2
|
+
|
|
3
|
+
[](https://github.com/quantifyearth/yirgacheffe/actions)
|
|
4
|
+
[](https://yirgacheffe.org)
|
|
5
|
+
[](https://pypi.org/project/yirgacheffe/)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
## Overview
|
|
9
|
+
|
|
10
|
+
Yirgacheffe is a declarative geospatial library, allowing you to operate on both raster and polygon geospatial datasets without having to do all the tedious book keeping around layer alignment or dealing with hardware concerns around memory or parallelism. you can load into memory safely.
|
|
11
|
+
|
|
12
|
+
Example common use-cases:
|
|
13
|
+
|
|
14
|
+
* Do the datasets overlap? Yirgacheffe will let you define either the intersection or the union of a set of different datasets, scaling up or down the area as required.
|
|
15
|
+
* Rasterisation of vector layers: if you have a vector dataset then you can add that to your computation and yirgaceffe will rasterize it on demand, so you never need to store more data in memory than necessary.
|
|
16
|
+
* Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
|
|
17
|
+
* Parallelisation of operations over many CPU cores.
|
|
18
|
+
* Built in support for optionally using GPUs via [MLX](https://ml-explore.github.io/mlx/build/html/index.html) support.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
Yirgacheffe is available via pypi, so can be installed with pip for example:
|
|
23
|
+
|
|
24
|
+
```SystemShell
|
|
25
|
+
$ pip install yirgacheffe
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Documentation
|
|
29
|
+
|
|
30
|
+
The documentation can be found on [yirgacheffe.org](https://yirgacheffe.org/)
|
|
31
|
+
|
|
32
|
+
## Simple examples:
|
|
33
|
+
|
|
34
|
+
Here is how to do cloud removal from [Sentinel-2 data](https://browser.dataspace.copernicus.eu/?zoom=14&lat=6.15468&lng=38.20581&themeId=DEFAULT-THEME&visualizationUrl=U2FsdGVkX1944lrmeTJcaSsnoxNMp4oucN1AjklGUANHd2cRZWyXnepHvzpaOWzMhH8SrWQo%2BqrOvOnu6f9FeCMrS%2FDZmvjzID%2FoE1tbOCEHK8ohPXjFqYojeR9%2B82ri&datasetId=S2_L2A_CDAS&fromTime=2025-09-09T00%3A00%3A00.000Z&toTime=2025-09-09T23%3A59%3A59.999Z&layerId=1_TRUE_COLOR&demSource3D=%22MAPZEN%22&cloudCoverage=30&dateMode=SINGLE), using the [Scene Classification Layer](https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/scene-classification/) data:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import yirgaceffe as yg
|
|
38
|
+
|
|
39
|
+
with (
|
|
40
|
+
yg.read_raster("T37NCG_20250909T073609_B06_20m.jp2") as vre2,
|
|
41
|
+
yg.read_raster("T37NCG_20250909T073609_SCL_20m.jp2") as scl,
|
|
42
|
+
):
|
|
43
|
+
is_cloud = (scl == 8) | (scl == 9) | (scl == 10) # various cloud types
|
|
44
|
+
is_shadow = (scl == 3)
|
|
45
|
+
is_bad = is_cloud | is_shadow
|
|
46
|
+
|
|
47
|
+
masked_vre2 = yg.where(is_bad, float("nan"), vre2)
|
|
48
|
+
masked_vre2.to_geotiff("vre2_cleaned.tif")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or a species' [Area of Habitat](https://www.sciencedirect.com/science/article/pii/S0169534719301892) calculation:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import yirgaceffe as yg
|
|
55
|
+
|
|
56
|
+
with (
|
|
57
|
+
yg.read_raster("habitats.tif") as habitat_map,
|
|
58
|
+
yg.read_raster('elevation.tif') as elevation_map,
|
|
59
|
+
yg.read_shape('species123.geojson') as range_map,
|
|
60
|
+
):
|
|
61
|
+
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
62
|
+
refined_elevation = (elevation_map >= species_min) & (elevation_map <= species_max)
|
|
63
|
+
aoh = refined_habitat * refined_elevation * range_polygon * area_per_pixel_map
|
|
64
|
+
print(f'Area of habitat: {aoh.sum()}')
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Citation
|
|
68
|
+
|
|
69
|
+
If you use Yirgacheffe in your research, please cite our paper:
|
|
70
|
+
|
|
71
|
+
> Michael Winston Dales, Alison Eyres, Patrick Ferris, Francesca A. Ridley, Simon Tarr, and Anil Madhavapeddy. 2025. Yirgacheffe: A Declarative Approach to Geospatial Data. In *Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet* (PROPL '25). Association for Computing Machinery, New York, NY, USA, 47–54. https://doi.org/10.1145/3759536.3763806
|
|
72
|
+
|
|
73
|
+
<details>
|
|
74
|
+
<summary>BibTeX</summary>
|
|
75
|
+
|
|
76
|
+
```bibtex
|
|
77
|
+
@inproceedings{10.1145/3759536.3763806,
|
|
78
|
+
author = {Dales, Michael Winston and Eyres, Alison and Ferris, Patrick and Ridley, Francesca A. and Tarr, Simon and Madhavapeddy, Anil},
|
|
79
|
+
title = {Yirgacheffe: A Declarative Approach to Geospatial Data},
|
|
80
|
+
year = {2025},
|
|
81
|
+
isbn = {9798400721618},
|
|
82
|
+
publisher = {Association for Computing Machinery},
|
|
83
|
+
address = {New York, NY, USA},
|
|
84
|
+
url = {https://doi.org/10.1145/3759536.3763806},
|
|
85
|
+
doi = {10.1145/3759536.3763806},
|
|
86
|
+
abstract = {We present Yirgacheffe, a declarative geospatial library that allows spatial algorithms to be implemented concisely, supports parallel execution, and avoids common errors by automatically handling data (large geospatial rasters) and resources (cores, memory, GPUs). Our primary user domain comprises ecologists, where a typical problem involves cleaning messy occurrence data, overlaying it over tiled rasters, combining layers, and deriving actionable insights from the results. We describe the successes of this approach towards driving key pipelines related to global biodiversity and describe the capability gaps that remain, hoping to motivate more research into geospatial domain-specific languages.},
|
|
87
|
+
booktitle = {Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet},
|
|
88
|
+
pages = {47–54},
|
|
89
|
+
numpages = {8},
|
|
90
|
+
keywords = {Biodiversity, Declarative, Geospatial, Python},
|
|
91
|
+
location = {Singapore, Singapore},
|
|
92
|
+
series = {PROPL '25}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
</details>
|
|
97
|
+
|
|
98
|
+
## Thanks
|
|
99
|
+
|
|
100
|
+
Thanks to discussion and feedback from my colleagues, particularly Alison Eyres, Patrick Ferris, Amelia Holcomb, and Anil Madhavapeddy.
|
|
101
|
+
|
|
102
|
+
Inspired by the work of Daniele Baisero in his AoH library.
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# pyproject.toml
|
|
2
|
+
|
|
3
|
+
[build-system]
|
|
4
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
5
|
+
build-backend = "setuptools.build_meta"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "yirgacheffe"
|
|
9
|
+
version = "1.10.2"
|
|
10
|
+
description = "Abstraction of gdal datasets for doing basic math operations"
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]
|
|
13
|
+
license = "ISC"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 5 - Production/Stable",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
22
|
+
]
|
|
23
|
+
keywords = ["gdal", "gis", "geospatial", "declarative"]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"numpy>=1.24,<3.0",
|
|
26
|
+
"gdal[numpy]>=3.8,<4.0",
|
|
27
|
+
"scikit-image>=0.20,<1.0",
|
|
28
|
+
"torch",
|
|
29
|
+
"dill",
|
|
30
|
+
"deprecation",
|
|
31
|
+
"tomli",
|
|
32
|
+
"h3",
|
|
33
|
+
"pyproj",
|
|
34
|
+
]
|
|
35
|
+
requires-python = ">=3.10"
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
mlx = [
|
|
39
|
+
"mlx",
|
|
40
|
+
]
|
|
41
|
+
matplotlib = [
|
|
42
|
+
"matplotlib",
|
|
43
|
+
]
|
|
44
|
+
dev = [
|
|
45
|
+
"mypy",
|
|
46
|
+
"pylint",
|
|
47
|
+
"pytest",
|
|
48
|
+
"pytest-cov",
|
|
49
|
+
"pytest-mock",
|
|
50
|
+
"build",
|
|
51
|
+
"twine",
|
|
52
|
+
"mkdocs-material",
|
|
53
|
+
"mkdocstrings-python",
|
|
54
|
+
"mike",
|
|
55
|
+
"mkdocs-gen-files",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
[project.urls]
|
|
59
|
+
Homepage = "https://yirgacheffe.org/"
|
|
60
|
+
Repository = "https://github.com/quantifyearth/yirgacheffe.git"
|
|
61
|
+
Issues = "https://github.com/quantifyearth/yirgacheffe/issues"
|
|
62
|
+
Changelog = "https://yirgacheffe.org/latest/changelog/"
|
|
63
|
+
|
|
64
|
+
[project.scripts]
|
|
65
|
+
realpython = "reader.__main__:main"
|
|
66
|
+
|
|
67
|
+
[tool.setuptools.package-data]
|
|
68
|
+
yirgacheffe = ["py.typed"]
|
|
69
|
+
|
|
70
|
+
[[tool.mypy.overrides]]
|
|
71
|
+
module = "osgeo.*"
|
|
72
|
+
ignore_missing_imports = true
|
|
73
|
+
|
|
74
|
+
[[tool.mypy.overrides]]
|
|
75
|
+
module = "h3.*"
|
|
76
|
+
ignore_missing_imports = true
|
|
77
|
+
|
|
78
|
+
[[tool.mypy.overrides]]
|
|
79
|
+
module = "deprecation.*"
|
|
80
|
+
ignore_missing_imports = true
|
|
81
|
+
|
|
82
|
+
[tool.setuptools.packages.find]
|
|
83
|
+
exclude = ["site*", "docs*", "tests*"]
|
|
@@ -4,8 +4,8 @@ import tempfile
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
|
-
import yirgacheffe
|
|
8
|
-
from tests.helpers import gdal_dataset_with_data,
|
|
7
|
+
import yirgacheffe as yg
|
|
8
|
+
from tests.helpers import gdal_dataset_with_data, make_vectors_with_multiple_ids
|
|
9
9
|
from yirgacheffe.layers import ConstantLayer, RasterLayer, VectorLayer
|
|
10
10
|
from yirgacheffe.window import Area
|
|
11
11
|
|
|
@@ -201,27 +201,91 @@ def test_constant_layer_result_lhs_multiply() -> None:
|
|
|
201
201
|
|
|
202
202
|
def test_vector_layers_add() -> None:
|
|
203
203
|
data1 = np.array([[1, 2], [3, 4]])
|
|
204
|
-
|
|
204
|
+
with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
|
|
205
|
+
with tempfile.TemporaryDirectory() as tempdir:
|
|
206
|
+
path = os.path.join(tempdir, "test.gpkg")
|
|
207
|
+
areas = {
|
|
208
|
+
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
209
|
+
(Area(0.0, 0.0, 10, -10), 43)
|
|
210
|
+
}
|
|
211
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
212
|
+
|
|
213
|
+
burn_value = 2
|
|
214
|
+
with VectorLayer.layer_from_file(
|
|
215
|
+
path,
|
|
216
|
+
None,
|
|
217
|
+
raster_layer.map_projection.scale,
|
|
218
|
+
raster_layer.map_projection.name,
|
|
219
|
+
burn_value=burn_value
|
|
220
|
+
) as vector_layer:
|
|
221
|
+
layer2_total = vector_layer.sum()
|
|
222
|
+
assert layer2_total == ((vector_layer.window.xsize * vector_layer.window.ysize) / 2) * burn_value
|
|
223
|
+
|
|
224
|
+
calc = raster_layer + vector_layer
|
|
225
|
+
|
|
226
|
+
assert calc.area == vector_layer.area
|
|
227
|
+
|
|
228
|
+
total = calc.sum()
|
|
229
|
+
assert total == layer2_total + np.sum(data1)
|
|
230
|
+
|
|
231
|
+
with RasterLayer.empty_raster_layer_like(calc) as result:
|
|
232
|
+
calc.save(result)
|
|
233
|
+
total = result.sum()
|
|
234
|
+
assert total == layer2_total + np.sum(data1)
|
|
235
|
+
|
|
236
|
+
def test_vector_layers_add_unbound_rhs() -> None:
|
|
237
|
+
data1 = np.array([[1, 2], [3, 4]])
|
|
238
|
+
with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
|
|
239
|
+
with tempfile.TemporaryDirectory() as tempdir:
|
|
240
|
+
path = os.path.join(tempdir, "test.gpkg")
|
|
241
|
+
areas = {
|
|
242
|
+
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
243
|
+
(Area(0.0, 0.0, 10, -10), 43)
|
|
244
|
+
}
|
|
245
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
205
246
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
210
|
-
(Area(0.0, 0.0, 10, -10), 43)
|
|
211
|
-
}
|
|
212
|
-
make_vectors_with_mutlile_ids(areas, path)
|
|
247
|
+
burn_value = 2
|
|
248
|
+
with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
|
|
249
|
+
calc = raster_layer + vector_layer
|
|
213
250
|
|
|
214
|
-
|
|
215
|
-
layer2 = VectorLayer.layer_from_file(path, None, layer1.pixel_scale, layer1.projection, burn_value=burn_value)
|
|
216
|
-
layer2_total = layer2.sum()
|
|
217
|
-
assert layer2_total == ((layer2.window.xsize * layer2.window.ysize) / 2) * burn_value
|
|
251
|
+
layer2_total = ((calc.window.xsize * calc.window.ysize) / 2) * burn_value
|
|
218
252
|
|
|
219
|
-
|
|
253
|
+
assert calc.area != vector_layer.area
|
|
220
254
|
|
|
221
|
-
|
|
255
|
+
total = calc.sum()
|
|
256
|
+
assert total == layer2_total + np.sum(data1)
|
|
257
|
+
|
|
258
|
+
with RasterLayer.empty_raster_layer_like(calc) as result:
|
|
259
|
+
calc.save(result)
|
|
260
|
+
total = result.sum()
|
|
261
|
+
assert total == layer2_total + np.sum(data1)
|
|
262
|
+
|
|
263
|
+
def test_vector_layers_add_unbound_lhs() -> None:
|
|
264
|
+
data1 = np.array([[1, 2], [3, 4]])
|
|
265
|
+
with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
|
|
266
|
+
with tempfile.TemporaryDirectory() as tempdir:
|
|
267
|
+
path = os.path.join(tempdir, "test.gpkg")
|
|
268
|
+
areas = {
|
|
269
|
+
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
270
|
+
(Area(0.0, 0.0, 10, -10), 43)
|
|
271
|
+
}
|
|
272
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
273
|
+
|
|
274
|
+
burn_value = 2
|
|
275
|
+
with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
|
|
276
|
+
calc = vector_layer + raster_layer
|
|
277
|
+
|
|
278
|
+
layer2_total = ((calc.window.xsize * calc.window.ysize) / 2) * burn_value
|
|
279
|
+
|
|
280
|
+
assert calc.area != vector_layer.area
|
|
281
|
+
|
|
282
|
+
total = calc.sum()
|
|
283
|
+
assert total == layer2_total + np.sum(data1)
|
|
222
284
|
|
|
223
|
-
|
|
224
|
-
|
|
285
|
+
with RasterLayer.empty_raster_layer_like(calc) as result:
|
|
286
|
+
calc.save(result)
|
|
287
|
+
total = result.sum()
|
|
288
|
+
assert total == layer2_total + np.sum(data1)
|
|
225
289
|
|
|
226
290
|
def test_vector_layers_multiply() -> None:
|
|
227
291
|
data1 = np.array([[1, 2], [3, 4]])
|
|
@@ -233,7 +297,7 @@ def test_vector_layers_multiply() -> None:
|
|
|
233
297
|
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
234
298
|
(Area(0.0, 0.0, 10, -10), 43)
|
|
235
299
|
}
|
|
236
|
-
|
|
300
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
237
301
|
|
|
238
302
|
burn_value = 2
|
|
239
303
|
layer2 = VectorLayer.layer_from_file(path, None, layer1.pixel_scale, layer1.projection, burn_value=burn_value)
|
|
@@ -251,7 +315,7 @@ def test_vector_layers_multiply() -> None:
|
|
|
251
315
|
expected = np.array([[2, 0], [0, 8]])
|
|
252
316
|
assert (expected == actual).all()
|
|
253
317
|
|
|
254
|
-
@pytest.mark.skipif(
|
|
318
|
+
@pytest.mark.skipif(yg._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
|
|
255
319
|
def test_parallel_save_windows() -> None:
|
|
256
320
|
data1 = np.array([[1, 2], [3, 4]])
|
|
257
321
|
data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120], [130, 140, 150, 160]])
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
|
|
2
2
|
import numpy as np
|
|
3
|
-
import
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import yirgacheffe as yg
|
|
4
6
|
from yirgacheffe.layers import RasterLayer, ConstantLayer
|
|
5
7
|
from yirgacheffe.operators import DataType
|
|
6
8
|
from yirgacheffe.window import Area, PixelScale
|
|
@@ -23,10 +25,27 @@ def test_constant_parallel_save(monkeypatch) -> None:
|
|
|
23
25
|
with RasterLayer.empty_raster_layer(area, scale, DataType.Float32) as result:
|
|
24
26
|
with ConstantLayer(42.0) as c:
|
|
25
27
|
with monkeypatch.context() as m:
|
|
26
|
-
m.setattr(
|
|
28
|
+
m.setattr(yg.constants, "YSTEP", 1)
|
|
27
29
|
c.parallel_save(result)
|
|
28
30
|
|
|
29
31
|
expected = np.full((20, 20), 42.0)
|
|
30
32
|
actual = result.read_array(0, 0, 20, 20)
|
|
31
33
|
|
|
32
34
|
assert (expected == actual).all()
|
|
35
|
+
|
|
36
|
+
@pytest.mark.parametrize("lhs,rhs,expected_equal", [
|
|
37
|
+
(1, 2, False),
|
|
38
|
+
(1, 1, True),
|
|
39
|
+
(1.0, 2.0, False),
|
|
40
|
+
(1.0, 1.0, True),
|
|
41
|
+
(1, 1.0, True), # This is Python standard behaviour
|
|
42
|
+
])
|
|
43
|
+
def test_cse_hash(lhs,rhs,expected_equal) -> None:
|
|
44
|
+
a = yg.constant(lhs)
|
|
45
|
+
b = yg.constant(rhs)
|
|
46
|
+
|
|
47
|
+
assert a is not b
|
|
48
|
+
assert a.name != b.name
|
|
49
|
+
|
|
50
|
+
are_hashed_same = a._cse_hash == b._cse_hash
|
|
51
|
+
assert expected_equal == are_hashed_same
|
|
@@ -22,7 +22,7 @@ from tests.helpers import gdal_dataset_with_data
|
|
|
22
22
|
])
|
|
23
23
|
def test_round_trip(gtype) -> None:
|
|
24
24
|
ytype = DataType.of_gdal(gtype)
|
|
25
|
-
backend_type = backend.
|
|
25
|
+
backend_type = backend.dtype_to_backend(ytype)
|
|
26
26
|
assert backend.backend_to_dtype(backend_type) == ytype
|
|
27
27
|
|
|
28
28
|
@pytest.mark.parametrize("ytype", [
|
|
@@ -41,10 +41,26 @@ def test_round_trip_from_gdal(ytype) -> None:
|
|
|
41
41
|
gtype = ytype.to_gdal()
|
|
42
42
|
assert DataType.of_gdal(gtype) == ytype
|
|
43
43
|
|
|
44
|
+
@pytest.mark.parametrize("ytype,expected", [
|
|
45
|
+
(DataType.Byte, 1),
|
|
46
|
+
(DataType.Int8, 1),
|
|
47
|
+
(DataType.Int16, 2),
|
|
48
|
+
(DataType.Int32, 4),
|
|
49
|
+
(DataType.Int64, 8),
|
|
50
|
+
(DataType.UInt8, 1),
|
|
51
|
+
(DataType.UInt16, 2),
|
|
52
|
+
(DataType.UInt32, 4),
|
|
53
|
+
(DataType.UInt64, 8),
|
|
54
|
+
(DataType.Float32, 4),
|
|
55
|
+
(DataType.Float64, 8),
|
|
56
|
+
])
|
|
57
|
+
def test_datatype_size(ytype,expected) -> None:
|
|
58
|
+
actual = ytype.sizeof()
|
|
59
|
+
assert expected == actual
|
|
60
|
+
|
|
44
61
|
def test_round_trip_float64() -> None:
|
|
45
|
-
backend_type = backend.
|
|
62
|
+
backend_type = backend.dtype_to_backend(DataType.Float64)
|
|
46
63
|
ytype = backend.backend_to_dtype(backend_type)
|
|
47
|
-
print(BACKEND, "sad")
|
|
48
64
|
match BACKEND:
|
|
49
65
|
case "NUMPY":
|
|
50
66
|
assert ytype == DataType.Float64
|
|
@@ -63,5 +79,51 @@ def test_float_to_int() -> None:
|
|
|
63
79
|
comp.save(result)
|
|
64
80
|
|
|
65
81
|
expected = backend.promote(np.array([[1, 2, 3, 4], [5, 6, 7, 8]]))
|
|
66
|
-
actual =
|
|
82
|
+
actual = result.read_array(0, 0, 4, 2)
|
|
67
83
|
assert (expected == actual).all()
|
|
84
|
+
|
|
85
|
+
@pytest.mark.parametrize("array,expected_type", [
|
|
86
|
+
(
|
|
87
|
+
np.ones((2, 2)).astype(np.int8),
|
|
88
|
+
DataType.Int8,
|
|
89
|
+
),
|
|
90
|
+
(
|
|
91
|
+
np.ones((2, 2)).astype(np.int16),
|
|
92
|
+
DataType.Int16,
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
np.ones((2, 2)).astype(np.int32),
|
|
96
|
+
DataType.Int32,
|
|
97
|
+
),
|
|
98
|
+
(
|
|
99
|
+
np.ones((2, 2)).astype(np.int64),
|
|
100
|
+
DataType.Int64,
|
|
101
|
+
),
|
|
102
|
+
(
|
|
103
|
+
np.ones((2, 2)).astype(np.uint8),
|
|
104
|
+
DataType.UInt8,
|
|
105
|
+
),
|
|
106
|
+
(
|
|
107
|
+
np.ones((2, 2)).astype(np.uint16),
|
|
108
|
+
DataType.UInt16,
|
|
109
|
+
),
|
|
110
|
+
(
|
|
111
|
+
np.ones((2, 2)).astype(np.uint32),
|
|
112
|
+
DataType.UInt32,
|
|
113
|
+
),
|
|
114
|
+
(
|
|
115
|
+
np.ones((2, 2)).astype(np.uint64),
|
|
116
|
+
DataType.UInt64,
|
|
117
|
+
),
|
|
118
|
+
(
|
|
119
|
+
np.ones((2, 2)).astype(np.float32),
|
|
120
|
+
DataType.Float32,
|
|
121
|
+
),
|
|
122
|
+
(
|
|
123
|
+
np.ones((2, 2)).astype(np.float64),
|
|
124
|
+
DataType.Float64,
|
|
125
|
+
),
|
|
126
|
+
])
|
|
127
|
+
def test_of_Array(array: np.ndarray, expected_type: DataType) -> None:
|
|
128
|
+
ytype = DataType.of_array(array)
|
|
129
|
+
assert ytype == expected_type
|