yirgacheffe 1.8.1__tar.gz → 1.10.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yirgacheffe-1.10.3/PKG-INFO +150 -0
- yirgacheffe-1.10.3/README.md +102 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/pyproject.toml +6 -1
- yirgacheffe-1.10.3/tests/test_aggregations.py +68 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_auto_windowing.py +5 -5
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_constants.py +23 -4
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_datatypes.py +64 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_h3layer.py +1 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_intersection.py +5 -5
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_multiband.py +7 -5
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_openers.py +31 -7
- yirgacheffe-1.10.3/tests/test_operator_hashing.py +372 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_operators.py +279 -47
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_parallel_operators.py +3 -3
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_pickle.py +1 -3
- yirgacheffe-1.10.3/tests/test_pixel_coord.py +257 -0
- yirgacheffe-1.10.3/tests/test_projection.py +45 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_raster.py +33 -0
- yirgacheffe-1.10.3/tests/test_reduce.py +24 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_rescaling.py +2 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_union.py +7 -7
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_vectors.py +11 -13
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_window.py +10 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/__init__.py +6 -1
- yirgacheffe-1.10.3/yirgacheffe/_backends/enumeration.py +150 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/_backends/mlx.py +24 -7
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/_backends/numpy.py +11 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/_core.py +56 -7
- yirgacheffe-1.8.1/yirgacheffe/_operators.py → yirgacheffe-1.10.3/yirgacheffe/_operators/__init__.py +540 -131
- yirgacheffe-1.10.3/yirgacheffe/_operators/cse.py +66 -0
- yirgacheffe-1.10.3/yirgacheffe/constants.py +8 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/area.py +2 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/base.py +6 -20
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/constant.py +4 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/group.py +5 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/h3layer.py +9 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/rasters.py +12 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/rescaled.py +11 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/vectors.py +13 -2
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/operators.py +1 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/rounding.py +2 -1
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/window.py +35 -10
- yirgacheffe-1.10.3/yirgacheffe.egg-info/PKG-INFO +150 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/SOURCES.txt +6 -2
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/requires.txt +5 -0
- yirgacheffe-1.8.1/PKG-INFO +0 -592
- yirgacheffe-1.8.1/README.md +0 -548
- yirgacheffe-1.8.1/tests/test_base.py +0 -189
- yirgacheffe-1.8.1/tests/test_projection.py +0 -32
- yirgacheffe-1.8.1/yirgacheffe/_backends/enumeration.py +0 -81
- yirgacheffe-1.8.1/yirgacheffe/constants.py +0 -8
- yirgacheffe-1.8.1/yirgacheffe.egg-info/PKG-INFO +0 -592
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/LICENSE +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/MANIFEST.in +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/setup.cfg +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_area.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_group.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_nodata.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_optimisation.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_rounding.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_save_with_window.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_sum_with_window.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/tests/test_uniform_area_layer.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/_backends/__init__.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/layers/__init__.py +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe/py.typed +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/dependency_links.txt +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/entry_points.txt +0 -0
- {yirgacheffe-1.8.1 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: yirgacheffe
|
|
3
|
+
Version: 1.10.3
|
|
4
|
+
Summary: Abstraction of gdal datasets for doing basic math operations
|
|
5
|
+
Author-email: Michael Dales <mwd24@cam.ac.uk>
|
|
6
|
+
License-Expression: ISC
|
|
7
|
+
Project-URL: Homepage, https://yirgacheffe.org/
|
|
8
|
+
Project-URL: Repository, https://github.com/quantifyearth/yirgacheffe.git
|
|
9
|
+
Project-URL: Issues, https://github.com/quantifyearth/yirgacheffe/issues
|
|
10
|
+
Project-URL: Changelog, https://yirgacheffe.org/latest/changelog/
|
|
11
|
+
Keywords: gdal,gis,geospatial,declarative
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: numpy<3.0,>=1.24
|
|
23
|
+
Requires-Dist: gdal[numpy]<4.0,>=3.8
|
|
24
|
+
Requires-Dist: scikit-image<1.0,>=0.20
|
|
25
|
+
Requires-Dist: torch
|
|
26
|
+
Requires-Dist: dill
|
|
27
|
+
Requires-Dist: deprecation
|
|
28
|
+
Requires-Dist: tomli
|
|
29
|
+
Requires-Dist: h3
|
|
30
|
+
Requires-Dist: pyproj
|
|
31
|
+
Provides-Extra: mlx
|
|
32
|
+
Requires-Dist: mlx; extra == "mlx"
|
|
33
|
+
Provides-Extra: matplotlib
|
|
34
|
+
Requires-Dist: matplotlib; extra == "matplotlib"
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: mypy; extra == "dev"
|
|
37
|
+
Requires-Dist: pylint; extra == "dev"
|
|
38
|
+
Requires-Dist: pytest; extra == "dev"
|
|
39
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
40
|
+
Requires-Dist: pytest-mock; extra == "dev"
|
|
41
|
+
Requires-Dist: build; extra == "dev"
|
|
42
|
+
Requires-Dist: twine; extra == "dev"
|
|
43
|
+
Requires-Dist: mkdocs-material; extra == "dev"
|
|
44
|
+
Requires-Dist: mkdocstrings-python; extra == "dev"
|
|
45
|
+
Requires-Dist: mike; extra == "dev"
|
|
46
|
+
Requires-Dist: mkdocs-gen-files; extra == "dev"
|
|
47
|
+
Dynamic: license-file
|
|
48
|
+
|
|
49
|
+
# Yirgacheffe: a declarative geospatial library for Python to make data-science with maps easier
|
|
50
|
+
|
|
51
|
+
[](https://github.com/quantifyearth/yirgacheffe/actions)
|
|
52
|
+
[](https://yirgacheffe.org)
|
|
53
|
+
[](https://pypi.org/project/yirgacheffe/)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
## Overview
|
|
57
|
+
|
|
58
|
+
Yirgacheffe is a declarative geospatial library, allowing you to operate on both raster and polygon geospatial datasets without having to do all the tedious book keeping around layer alignment or dealing with hardware concerns around memory or parallelism. you can load into memory safely.
|
|
59
|
+
|
|
60
|
+
Example common use-cases:
|
|
61
|
+
|
|
62
|
+
* Do the datasets overlap? Yirgacheffe will let you define either the intersection or the union of a set of different datasets, scaling up or down the area as required.
|
|
63
|
+
* Rasterisation of vector layers: if you have a vector dataset then you can add that to your computation and yirgaceffe will rasterize it on demand, so you never need to store more data in memory than necessary.
|
|
64
|
+
* Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
|
|
65
|
+
* Parallelisation of operations over many CPU cores.
|
|
66
|
+
* Built in support for optionally using GPUs via [MLX](https://ml-explore.github.io/mlx/build/html/index.html) support.
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
Yirgacheffe is available via pypi, so can be installed with pip for example:
|
|
71
|
+
|
|
72
|
+
```SystemShell
|
|
73
|
+
$ pip install yirgacheffe
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Documentation
|
|
77
|
+
|
|
78
|
+
The documentation can be found on [yirgacheffe.org](https://yirgacheffe.org/)
|
|
79
|
+
|
|
80
|
+
## Simple examples:
|
|
81
|
+
|
|
82
|
+
Here is how to do cloud removal from [Sentinel-2 data](https://browser.dataspace.copernicus.eu/?zoom=14&lat=6.15468&lng=38.20581&themeId=DEFAULT-THEME&visualizationUrl=U2FsdGVkX1944lrmeTJcaSsnoxNMp4oucN1AjklGUANHd2cRZWyXnepHvzpaOWzMhH8SrWQo%2BqrOvOnu6f9FeCMrS%2FDZmvjzID%2FoE1tbOCEHK8ohPXjFqYojeR9%2B82ri&datasetId=S2_L2A_CDAS&fromTime=2025-09-09T00%3A00%3A00.000Z&toTime=2025-09-09T23%3A59%3A59.999Z&layerId=1_TRUE_COLOR&demSource3D=%22MAPZEN%22&cloudCoverage=30&dateMode=SINGLE), using the [Scene Classification Layer](https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/scene-classification/) data:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import yirgaceffe as yg
|
|
86
|
+
|
|
87
|
+
with (
|
|
88
|
+
yg.read_raster("T37NCG_20250909T073609_B06_20m.jp2") as vre2,
|
|
89
|
+
yg.read_raster("T37NCG_20250909T073609_SCL_20m.jp2") as scl,
|
|
90
|
+
):
|
|
91
|
+
is_cloud = (scl == 8) | (scl == 9) | (scl == 10) # various cloud types
|
|
92
|
+
is_shadow = (scl == 3)
|
|
93
|
+
is_bad = is_cloud | is_shadow
|
|
94
|
+
|
|
95
|
+
masked_vre2 = yg.where(is_bad, float("nan"), vre2)
|
|
96
|
+
masked_vre2.to_geotiff("vre2_cleaned.tif")
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
or a species' [Area of Habitat](https://www.sciencedirect.com/science/article/pii/S0169534719301892) calculation:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import yirgaceffe as yg
|
|
103
|
+
|
|
104
|
+
with (
|
|
105
|
+
yg.read_raster("habitats.tif") as habitat_map,
|
|
106
|
+
yg.read_raster('elevation.tif') as elevation_map,
|
|
107
|
+
yg.read_shape('species123.geojson') as range_map,
|
|
108
|
+
):
|
|
109
|
+
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
110
|
+
refined_elevation = (elevation_map >= species_min) & (elevation_map <= species_max)
|
|
111
|
+
aoh = refined_habitat * refined_elevation * range_polygon * area_per_pixel_map
|
|
112
|
+
print(f'Area of habitat: {aoh.sum()}')
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Citation
|
|
116
|
+
|
|
117
|
+
If you use Yirgacheffe in your research, please cite our paper:
|
|
118
|
+
|
|
119
|
+
> Michael Winston Dales, Alison Eyres, Patrick Ferris, Francesca A. Ridley, Simon Tarr, and Anil Madhavapeddy. 2025. Yirgacheffe: A Declarative Approach to Geospatial Data. In *Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet* (PROPL '25). Association for Computing Machinery, New York, NY, USA, 47–54. https://doi.org/10.1145/3759536.3763806
|
|
120
|
+
|
|
121
|
+
<details>
|
|
122
|
+
<summary>BibTeX</summary>
|
|
123
|
+
|
|
124
|
+
```bibtex
|
|
125
|
+
@inproceedings{10.1145/3759536.3763806,
|
|
126
|
+
author = {Dales, Michael Winston and Eyres, Alison and Ferris, Patrick and Ridley, Francesca A. and Tarr, Simon and Madhavapeddy, Anil},
|
|
127
|
+
title = {Yirgacheffe: A Declarative Approach to Geospatial Data},
|
|
128
|
+
year = {2025},
|
|
129
|
+
isbn = {9798400721618},
|
|
130
|
+
publisher = {Association for Computing Machinery},
|
|
131
|
+
address = {New York, NY, USA},
|
|
132
|
+
url = {https://doi.org/10.1145/3759536.3763806},
|
|
133
|
+
doi = {10.1145/3759536.3763806},
|
|
134
|
+
abstract = {We present Yirgacheffe, a declarative geospatial library that allows spatial algorithms to be implemented concisely, supports parallel execution, and avoids common errors by automatically handling data (large geospatial rasters) and resources (cores, memory, GPUs). Our primary user domain comprises ecologists, where a typical problem involves cleaning messy occurrence data, overlaying it over tiled rasters, combining layers, and deriving actionable insights from the results. We describe the successes of this approach towards driving key pipelines related to global biodiversity and describe the capability gaps that remain, hoping to motivate more research into geospatial domain-specific languages.},
|
|
135
|
+
booktitle = {Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet},
|
|
136
|
+
pages = {47–54},
|
|
137
|
+
numpages = {8},
|
|
138
|
+
keywords = {Biodiversity, Declarative, Geospatial, Python},
|
|
139
|
+
location = {Singapore, Singapore},
|
|
140
|
+
series = {PROPL '25}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
</details>
|
|
145
|
+
|
|
146
|
+
## Thanks
|
|
147
|
+
|
|
148
|
+
Thanks to discussion and feedback from my colleagues, particularly Alison Eyres, Patrick Ferris, Amelia Holcomb, and Anil Madhavapeddy.
|
|
149
|
+
|
|
150
|
+
Inspired by the work of Daniele Baisero in his AoH library.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Yirgacheffe: a declarative geospatial library for Python to make data-science with maps easier
|
|
2
|
+
|
|
3
|
+
[](https://github.com/quantifyearth/yirgacheffe/actions)
|
|
4
|
+
[](https://yirgacheffe.org)
|
|
5
|
+
[](https://pypi.org/project/yirgacheffe/)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
## Overview
|
|
9
|
+
|
|
10
|
+
Yirgacheffe is a declarative geospatial library, allowing you to operate on both raster and polygon geospatial datasets without having to do all the tedious book keeping around layer alignment or dealing with hardware concerns around memory or parallelism. you can load into memory safely.
|
|
11
|
+
|
|
12
|
+
Example common use-cases:
|
|
13
|
+
|
|
14
|
+
* Do the datasets overlap? Yirgacheffe will let you define either the intersection or the union of a set of different datasets, scaling up or down the area as required.
|
|
15
|
+
* Rasterisation of vector layers: if you have a vector dataset then you can add that to your computation and yirgaceffe will rasterize it on demand, so you never need to store more data in memory than necessary.
|
|
16
|
+
* Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
|
|
17
|
+
* Parallelisation of operations over many CPU cores.
|
|
18
|
+
* Built in support for optionally using GPUs via [MLX](https://ml-explore.github.io/mlx/build/html/index.html) support.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
Yirgacheffe is available via pypi, so can be installed with pip for example:
|
|
23
|
+
|
|
24
|
+
```SystemShell
|
|
25
|
+
$ pip install yirgacheffe
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Documentation
|
|
29
|
+
|
|
30
|
+
The documentation can be found on [yirgacheffe.org](https://yirgacheffe.org/)
|
|
31
|
+
|
|
32
|
+
## Simple examples:
|
|
33
|
+
|
|
34
|
+
Here is how to do cloud removal from [Sentinel-2 data](https://browser.dataspace.copernicus.eu/?zoom=14&lat=6.15468&lng=38.20581&themeId=DEFAULT-THEME&visualizationUrl=U2FsdGVkX1944lrmeTJcaSsnoxNMp4oucN1AjklGUANHd2cRZWyXnepHvzpaOWzMhH8SrWQo%2BqrOvOnu6f9FeCMrS%2FDZmvjzID%2FoE1tbOCEHK8ohPXjFqYojeR9%2B82ri&datasetId=S2_L2A_CDAS&fromTime=2025-09-09T00%3A00%3A00.000Z&toTime=2025-09-09T23%3A59%3A59.999Z&layerId=1_TRUE_COLOR&demSource3D=%22MAPZEN%22&cloudCoverage=30&dateMode=SINGLE), using the [Scene Classification Layer](https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/scene-classification/) data:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import yirgaceffe as yg
|
|
38
|
+
|
|
39
|
+
with (
|
|
40
|
+
yg.read_raster("T37NCG_20250909T073609_B06_20m.jp2") as vre2,
|
|
41
|
+
yg.read_raster("T37NCG_20250909T073609_SCL_20m.jp2") as scl,
|
|
42
|
+
):
|
|
43
|
+
is_cloud = (scl == 8) | (scl == 9) | (scl == 10) # various cloud types
|
|
44
|
+
is_shadow = (scl == 3)
|
|
45
|
+
is_bad = is_cloud | is_shadow
|
|
46
|
+
|
|
47
|
+
masked_vre2 = yg.where(is_bad, float("nan"), vre2)
|
|
48
|
+
masked_vre2.to_geotiff("vre2_cleaned.tif")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
or a species' [Area of Habitat](https://www.sciencedirect.com/science/article/pii/S0169534719301892) calculation:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import yirgaceffe as yg
|
|
55
|
+
|
|
56
|
+
with (
|
|
57
|
+
yg.read_raster("habitats.tif") as habitat_map,
|
|
58
|
+
yg.read_raster('elevation.tif') as elevation_map,
|
|
59
|
+
yg.read_shape('species123.geojson') as range_map,
|
|
60
|
+
):
|
|
61
|
+
refined_habitat = habitat_map.isin([...species habitat codes...])
|
|
62
|
+
refined_elevation = (elevation_map >= species_min) & (elevation_map <= species_max)
|
|
63
|
+
aoh = refined_habitat * refined_elevation * range_polygon * area_per_pixel_map
|
|
64
|
+
print(f'Area of habitat: {aoh.sum()}')
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Citation
|
|
68
|
+
|
|
69
|
+
If you use Yirgacheffe in your research, please cite our paper:
|
|
70
|
+
|
|
71
|
+
> Michael Winston Dales, Alison Eyres, Patrick Ferris, Francesca A. Ridley, Simon Tarr, and Anil Madhavapeddy. 2025. Yirgacheffe: A Declarative Approach to Geospatial Data. In *Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet* (PROPL '25). Association for Computing Machinery, New York, NY, USA, 47–54. https://doi.org/10.1145/3759536.3763806
|
|
72
|
+
|
|
73
|
+
<details>
|
|
74
|
+
<summary>BibTeX</summary>
|
|
75
|
+
|
|
76
|
+
```bibtex
|
|
77
|
+
@inproceedings{10.1145/3759536.3763806,
|
|
78
|
+
author = {Dales, Michael Winston and Eyres, Alison and Ferris, Patrick and Ridley, Francesca A. and Tarr, Simon and Madhavapeddy, Anil},
|
|
79
|
+
title = {Yirgacheffe: A Declarative Approach to Geospatial Data},
|
|
80
|
+
year = {2025},
|
|
81
|
+
isbn = {9798400721618},
|
|
82
|
+
publisher = {Association for Computing Machinery},
|
|
83
|
+
address = {New York, NY, USA},
|
|
84
|
+
url = {https://doi.org/10.1145/3759536.3763806},
|
|
85
|
+
doi = {10.1145/3759536.3763806},
|
|
86
|
+
abstract = {We present Yirgacheffe, a declarative geospatial library that allows spatial algorithms to be implemented concisely, supports parallel execution, and avoids common errors by automatically handling data (large geospatial rasters) and resources (cores, memory, GPUs). Our primary user domain comprises ecologists, where a typical problem involves cleaning messy occurrence data, overlaying it over tiled rasters, combining layers, and deriving actionable insights from the results. We describe the successes of this approach towards driving key pipelines related to global biodiversity and describe the capability gaps that remain, hoping to motivate more research into geospatial domain-specific languages.},
|
|
87
|
+
booktitle = {Proceedings of the 2nd ACM SIGPLAN International Workshop on Programming for the Planet},
|
|
88
|
+
pages = {47–54},
|
|
89
|
+
numpages = {8},
|
|
90
|
+
keywords = {Biodiversity, Declarative, Geospatial, Python},
|
|
91
|
+
location = {Singapore, Singapore},
|
|
92
|
+
series = {PROPL '25}
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
</details>
|
|
97
|
+
|
|
98
|
+
## Thanks
|
|
99
|
+
|
|
100
|
+
Thanks to discussion and feedback from my colleagues, particularly Alison Eyres, Patrick Ferris, Amelia Holcomb, and Anil Madhavapeddy.
|
|
101
|
+
|
|
102
|
+
Inspired by the work of Daniele Baisero in his AoH library.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "yirgacheffe"
|
|
9
|
-
version = "1.
|
|
9
|
+
version = "1.10.3"
|
|
10
10
|
description = "Abstraction of gdal datasets for doing basic math operations"
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]
|
|
@@ -30,6 +30,7 @@ dependencies = [
|
|
|
30
30
|
"deprecation",
|
|
31
31
|
"tomli",
|
|
32
32
|
"h3",
|
|
33
|
+
"pyproj",
|
|
33
34
|
]
|
|
34
35
|
requires-python = ">=3.10"
|
|
35
36
|
|
|
@@ -37,11 +38,15 @@ requires-python = ">=3.10"
|
|
|
37
38
|
mlx = [
|
|
38
39
|
"mlx",
|
|
39
40
|
]
|
|
41
|
+
matplotlib = [
|
|
42
|
+
"matplotlib",
|
|
43
|
+
]
|
|
40
44
|
dev = [
|
|
41
45
|
"mypy",
|
|
42
46
|
"pylint",
|
|
43
47
|
"pytest",
|
|
44
48
|
"pytest-cov",
|
|
49
|
+
"pytest-mock",
|
|
45
50
|
"build",
|
|
46
51
|
"twine",
|
|
47
52
|
"mkdocs-material",
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
import yirgacheffe as yg
|
|
7
|
+
from yirgacheffe._backends import backend, BACKEND
|
|
8
|
+
|
|
9
|
+
def test_sum_result_is_scalar() -> None:
|
|
10
|
+
rng = np.random.default_rng(seed=42)
|
|
11
|
+
data = rng.integers(0, 128, size=(1000, 1000))
|
|
12
|
+
with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
|
|
13
|
+
total = layer.sum()
|
|
14
|
+
json.dumps(total)
|
|
15
|
+
|
|
16
|
+
@pytest.mark.parametrize("c,dtype,maxval", [
|
|
17
|
+
(int(2), np.int8, 120),
|
|
18
|
+
(int(2), np.uint8, 250),
|
|
19
|
+
(int(2), np.int16, 32000),
|
|
20
|
+
(int(2), np.uint16, 64000),
|
|
21
|
+
(int(2), np.int32, 66000),
|
|
22
|
+
(int(2), np.uint32, 66000),
|
|
23
|
+
])
|
|
24
|
+
@pytest.mark.parametrize("step", [1, 2, 4, 8])
|
|
25
|
+
def test_sums_of_calc_int(monkeypatch, step, c, dtype: type, maxval: int) -> None:
|
|
26
|
+
with monkeypatch.context() as m:
|
|
27
|
+
m.setattr(yg.constants, "YSTEP", step)
|
|
28
|
+
|
|
29
|
+
rng = np.random.default_rng(seed=42)
|
|
30
|
+
|
|
31
|
+
data = rng.integers(0, maxval, size=(1000, 1000), dtype=dtype)
|
|
32
|
+
typed_data = backend.promote(data)
|
|
33
|
+
|
|
34
|
+
assert np.sum(data) == backend.sum_op(typed_data)
|
|
35
|
+
|
|
36
|
+
with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
|
|
37
|
+
assert layer.sum() == backend.sum_op(typed_data)
|
|
38
|
+
calc = layer * c
|
|
39
|
+
actual = calc.sum()
|
|
40
|
+
expected = backend.sum_op(typed_data * c)
|
|
41
|
+
assert actual == expected
|
|
42
|
+
|
|
43
|
+
@pytest.mark.parametrize("c,dtype,maxval", [
|
|
44
|
+
(float(2.5), np.int8, 120),
|
|
45
|
+
(float(2.5), np.uint8, 250),
|
|
46
|
+
(float(2.5), np.int16, 32000),
|
|
47
|
+
(float(2.5), np.uint16, 640),
|
|
48
|
+
(float(2.5), np.int32, 660),
|
|
49
|
+
(float(2.5), np.uint32, 660),
|
|
50
|
+
])
|
|
51
|
+
@pytest.mark.parametrize("step", [1, 2, 4, 8])
|
|
52
|
+
def test_sums_of_calc_float_mlx(monkeypatch, step, c, dtype: type, maxval: int) -> None:
|
|
53
|
+
with monkeypatch.context() as m:
|
|
54
|
+
m.setattr(yg.constants, "YSTEP", step)
|
|
55
|
+
|
|
56
|
+
rng = np.random.default_rng(seed=42)
|
|
57
|
+
|
|
58
|
+
data = rng.integers(0, maxval, size=(100, 100), dtype=dtype)
|
|
59
|
+
typed_data = backend.promote(data)
|
|
60
|
+
|
|
61
|
+
with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
|
|
62
|
+
assert layer.sum() == backend.sum_op(typed_data)
|
|
63
|
+
calc = layer * c
|
|
64
|
+
actual = calc.sum()
|
|
65
|
+
expected = backend.sum_op(typed_data * c)
|
|
66
|
+
# MLX has a maximum float size of 32 bit for GPU and NUMPY has 64 bit on CPU
|
|
67
|
+
rel = 1e-6 if BACKEND == "MLX" else 1e-10
|
|
68
|
+
assert float(actual) == pytest.approx(float(expected), rel=rel)
|
|
@@ -5,7 +5,7 @@ import numpy as np
|
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
7
|
import yirgacheffe as yg
|
|
8
|
-
from tests.helpers import gdal_dataset_with_data,
|
|
8
|
+
from tests.helpers import gdal_dataset_with_data, make_vectors_with_multiple_ids
|
|
9
9
|
from yirgacheffe.layers import ConstantLayer, RasterLayer, VectorLayer
|
|
10
10
|
from yirgacheffe.window import Area
|
|
11
11
|
|
|
@@ -208,7 +208,7 @@ def test_vector_layers_add() -> None:
|
|
|
208
208
|
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
209
209
|
(Area(0.0, 0.0, 10, -10), 43)
|
|
210
210
|
}
|
|
211
|
-
|
|
211
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
212
212
|
|
|
213
213
|
burn_value = 2
|
|
214
214
|
with VectorLayer.layer_from_file(
|
|
@@ -242,7 +242,7 @@ def test_vector_layers_add_unbound_rhs() -> None:
|
|
|
242
242
|
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
243
243
|
(Area(0.0, 0.0, 10, -10), 43)
|
|
244
244
|
}
|
|
245
|
-
|
|
245
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
246
246
|
|
|
247
247
|
burn_value = 2
|
|
248
248
|
with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
|
|
@@ -269,7 +269,7 @@ def test_vector_layers_add_unbound_lhs() -> None:
|
|
|
269
269
|
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
270
270
|
(Area(0.0, 0.0, 10, -10), 43)
|
|
271
271
|
}
|
|
272
|
-
|
|
272
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
273
273
|
|
|
274
274
|
burn_value = 2
|
|
275
275
|
with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
|
|
@@ -297,7 +297,7 @@ def test_vector_layers_multiply() -> None:
|
|
|
297
297
|
(Area(-10.0, 10.0, 0.0, 0.0), 42),
|
|
298
298
|
(Area(0.0, 0.0, 10, -10), 43)
|
|
299
299
|
}
|
|
300
|
-
|
|
300
|
+
make_vectors_with_multiple_ids(areas, path)
|
|
301
301
|
|
|
302
302
|
burn_value = 2
|
|
303
303
|
layer2 = VectorLayer.layer_from_file(path, None, layer1.pixel_scale, layer1.projection, burn_value=burn_value)
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
|
|
2
2
|
import numpy as np
|
|
3
|
-
import
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
import yirgacheffe as yg
|
|
4
6
|
from yirgacheffe.layers import RasterLayer, ConstantLayer
|
|
5
7
|
from yirgacheffe.operators import DataType
|
|
6
8
|
from yirgacheffe.window import Area, PixelScale
|
|
@@ -21,12 +23,29 @@ def test_constant_parallel_save(monkeypatch) -> None:
|
|
|
21
23
|
area = Area(left=-1.0, right=1.0, top=1.0, bottom=-1.0)
|
|
22
24
|
scale = PixelScale(0.1, -0.1)
|
|
23
25
|
with RasterLayer.empty_raster_layer(area, scale, DataType.Float32) as result:
|
|
24
|
-
with
|
|
25
|
-
|
|
26
|
-
|
|
26
|
+
with monkeypatch.context() as m:
|
|
27
|
+
m.setattr(yg.constants, "YSTEP", 1)
|
|
28
|
+
with ConstantLayer(42.0) as c:
|
|
27
29
|
c.parallel_save(result)
|
|
28
30
|
|
|
29
31
|
expected = np.full((20, 20), 42.0)
|
|
30
32
|
actual = result.read_array(0, 0, 20, 20)
|
|
31
33
|
|
|
32
34
|
assert (expected == actual).all()
|
|
35
|
+
|
|
36
|
+
@pytest.mark.parametrize("lhs,rhs,expected_equal", [
|
|
37
|
+
(1, 2, False),
|
|
38
|
+
(1, 1, True),
|
|
39
|
+
(1.0, 2.0, False),
|
|
40
|
+
(1.0, 1.0, True),
|
|
41
|
+
(1, 1.0, True), # This is Python standard behaviour
|
|
42
|
+
])
|
|
43
|
+
def test_cse_hash(lhs,rhs,expected_equal) -> None:
|
|
44
|
+
a = yg.constant(lhs)
|
|
45
|
+
b = yg.constant(rhs)
|
|
46
|
+
|
|
47
|
+
assert a is not b
|
|
48
|
+
assert a.name != b.name
|
|
49
|
+
|
|
50
|
+
are_hashed_same = a._cse_hash == b._cse_hash
|
|
51
|
+
assert expected_equal == are_hashed_same
|
|
@@ -41,6 +41,23 @@ def test_round_trip_from_gdal(ytype) -> None:
|
|
|
41
41
|
gtype = ytype.to_gdal()
|
|
42
42
|
assert DataType.of_gdal(gtype) == ytype
|
|
43
43
|
|
|
44
|
+
@pytest.mark.parametrize("ytype,expected", [
|
|
45
|
+
(DataType.Byte, 1),
|
|
46
|
+
(DataType.Int8, 1),
|
|
47
|
+
(DataType.Int16, 2),
|
|
48
|
+
(DataType.Int32, 4),
|
|
49
|
+
(DataType.Int64, 8),
|
|
50
|
+
(DataType.UInt8, 1),
|
|
51
|
+
(DataType.UInt16, 2),
|
|
52
|
+
(DataType.UInt32, 4),
|
|
53
|
+
(DataType.UInt64, 8),
|
|
54
|
+
(DataType.Float32, 4),
|
|
55
|
+
(DataType.Float64, 8),
|
|
56
|
+
])
|
|
57
|
+
def test_datatype_size(ytype,expected) -> None:
|
|
58
|
+
actual = ytype.sizeof()
|
|
59
|
+
assert expected == actual
|
|
60
|
+
|
|
44
61
|
def test_round_trip_float64() -> None:
|
|
45
62
|
backend_type = backend.dtype_to_backend(DataType.Float64)
|
|
46
63
|
ytype = backend.backend_to_dtype(backend_type)
|
|
@@ -62,5 +79,51 @@ def test_float_to_int() -> None:
|
|
|
62
79
|
comp.save(result)
|
|
63
80
|
|
|
64
81
|
expected = backend.promote(np.array([[1, 2, 3, 4], [5, 6, 7, 8]]))
|
|
65
|
-
actual =
|
|
82
|
+
actual = result.read_array(0, 0, 4, 2)
|
|
66
83
|
assert (expected == actual).all()
|
|
84
|
+
|
|
85
|
+
@pytest.mark.parametrize("array,expected_type", [
|
|
86
|
+
(
|
|
87
|
+
np.ones((2, 2)).astype(np.int8),
|
|
88
|
+
DataType.Int8,
|
|
89
|
+
),
|
|
90
|
+
(
|
|
91
|
+
np.ones((2, 2)).astype(np.int16),
|
|
92
|
+
DataType.Int16,
|
|
93
|
+
),
|
|
94
|
+
(
|
|
95
|
+
np.ones((2, 2)).astype(np.int32),
|
|
96
|
+
DataType.Int32,
|
|
97
|
+
),
|
|
98
|
+
(
|
|
99
|
+
np.ones((2, 2)).astype(np.int64),
|
|
100
|
+
DataType.Int64,
|
|
101
|
+
),
|
|
102
|
+
(
|
|
103
|
+
np.ones((2, 2)).astype(np.uint8),
|
|
104
|
+
DataType.UInt8,
|
|
105
|
+
),
|
|
106
|
+
(
|
|
107
|
+
np.ones((2, 2)).astype(np.uint16),
|
|
108
|
+
DataType.UInt16,
|
|
109
|
+
),
|
|
110
|
+
(
|
|
111
|
+
np.ones((2, 2)).astype(np.uint32),
|
|
112
|
+
DataType.UInt32,
|
|
113
|
+
),
|
|
114
|
+
(
|
|
115
|
+
np.ones((2, 2)).astype(np.uint64),
|
|
116
|
+
DataType.UInt64,
|
|
117
|
+
),
|
|
118
|
+
(
|
|
119
|
+
np.ones((2, 2)).astype(np.float32),
|
|
120
|
+
DataType.Float32,
|
|
121
|
+
),
|
|
122
|
+
(
|
|
123
|
+
np.ones((2, 2)).astype(np.float64),
|
|
124
|
+
DataType.Float64,
|
|
125
|
+
),
|
|
126
|
+
])
|
|
127
|
+
def test_of_Array(array: np.ndarray, expected_type: DataType) -> None:
|
|
128
|
+
ytype = DataType.of_array(array)
|
|
129
|
+
assert ytype == expected_type
|
|
@@ -23,7 +23,7 @@ def test_h3_layer(cell_id: str, is_valid: bool, expected_zoom: int) -> None:
|
|
|
23
23
|
if is_valid:
|
|
24
24
|
with H3CellLayer(cell_id, MapProjection(WGS_84_PROJECTION, 0.001, -0.001)) as layer:
|
|
25
25
|
assert layer.zoom == expected_zoom
|
|
26
|
-
assert layer.
|
|
26
|
+
assert layer.map_projection.epsg == 4326
|
|
27
27
|
|
|
28
28
|
# without getting too deep, we'd expect a mix of zeros and ones in the data
|
|
29
29
|
window = layer.window
|
|
@@ -67,7 +67,7 @@ def test_find_intersection_with_vector_unbound() -> None:
|
|
|
67
67
|
path = Path(tempdir) / "test.gpkg"
|
|
68
68
|
area = Area(left=58, top=74, right=180, bottom=42)
|
|
69
69
|
make_vectors_with_id(42, {area}, path)
|
|
70
|
-
assert path.exists
|
|
70
|
+
assert path.exists()
|
|
71
71
|
|
|
72
72
|
raster = RasterLayer(gdal_dataset_of_region(Area(left=-180.05, top=90.09, right=180.05, bottom=-90.09), 0.13))
|
|
73
73
|
vector = VectorLayer.layer_from_file(path, None, None, None)
|
|
@@ -86,10 +86,10 @@ def test_find_intersection_with_vector_bound() -> None:
|
|
|
86
86
|
path = Path(tempdir) / "test.gpkg"
|
|
87
87
|
area = Area(left=58, top=74, right=180, bottom=42)
|
|
88
88
|
make_vectors_with_id(42, {area}, path)
|
|
89
|
-
assert path.exists
|
|
89
|
+
assert path.exists()
|
|
90
90
|
|
|
91
91
|
raster = RasterLayer(gdal_dataset_of_region(Area(left=-180.05, top=90.09, right=180.05, bottom=-90.09), 0.13))
|
|
92
|
-
vector = VectorLayer.
|
|
92
|
+
vector = VectorLayer.layer_from_file_like(path, raster)
|
|
93
93
|
assert vector.area != area
|
|
94
94
|
|
|
95
95
|
layers = [raster, vector]
|
|
@@ -104,10 +104,10 @@ def test_find_intersection_with_vector_awkward_rounding() -> None:
|
|
|
104
104
|
path = Path(tempdir) / "test.gpkg"
|
|
105
105
|
area = Area(left=-90, top=45, right=90, bottom=-45)
|
|
106
106
|
make_vectors_with_id(42, {area}, path)
|
|
107
|
-
assert path.exists
|
|
107
|
+
assert path.exists()
|
|
108
108
|
|
|
109
109
|
raster = RasterLayer(gdal_dataset_of_region(Area(left=-180, top=90, right=180, bottom=-90), 18.0))
|
|
110
|
-
vector = VectorLayer.
|
|
110
|
+
vector = VectorLayer.layer_from_file_like(path, raster)
|
|
111
111
|
|
|
112
112
|
rounded_area = Area(left=-90, top=54, right=90, bottom=-54)
|
|
113
113
|
assert vector.area == rounded_area
|
|
@@ -4,7 +4,7 @@ import tempfile
|
|
|
4
4
|
import numpy as np
|
|
5
5
|
from osgeo import gdal
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import yirgacheffe as yg
|
|
8
8
|
from yirgacheffe.layers import RasterLayer
|
|
9
9
|
from yirgacheffe.window import Area, PixelScale
|
|
10
10
|
|
|
@@ -23,9 +23,10 @@ def test_simple_two_band_image() -> None:
|
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
# Create a set of rasters in turn to fill each band
|
|
26
|
+
projection = yg.MapProjection("epsg:4326", 1.0, -1.0)
|
|
26
27
|
for i in range(bands):
|
|
27
28
|
data1 = np.full((2, 2), i+1)
|
|
28
|
-
layer1 =
|
|
29
|
+
layer1 = yg.from_array(data1, (-1.0, 1.0), projection)
|
|
29
30
|
layer1.save(target, band=i+1)
|
|
30
31
|
|
|
31
32
|
# force things to disk
|
|
@@ -33,7 +34,7 @@ def test_simple_two_band_image() -> None:
|
|
|
33
34
|
|
|
34
35
|
#check they do what we expect
|
|
35
36
|
for i in range(bands):
|
|
36
|
-
o =
|
|
37
|
+
o = yg.read_raster(target_path, band=i+1)
|
|
37
38
|
assert o.sum() == (4 * (i + 1))
|
|
38
39
|
|
|
39
40
|
def test_stack_tifs_with_area_match() -> None:
|
|
@@ -43,9 +44,10 @@ def test_stack_tifs_with_area_match() -> None:
|
|
|
43
44
|
# slight alignment offset when we create them)
|
|
44
45
|
bands = 4
|
|
45
46
|
source_layers = []
|
|
47
|
+
projection = yg.MapProjection("epsg:4326", 1.0, -1.0)
|
|
46
48
|
for i in range(bands):
|
|
47
49
|
data1 = np.full((100, 100), i+1)
|
|
48
|
-
layer1 =
|
|
50
|
+
layer1 = yg.from_array(data1, (-100+i, 100+i), projection)
|
|
49
51
|
source_layers.append(layer1)
|
|
50
52
|
|
|
51
53
|
intersection = RasterLayer.find_intersection(source_layers)
|
|
@@ -66,5 +68,5 @@ def test_stack_tifs_with_area_match() -> None:
|
|
|
66
68
|
|
|
67
69
|
#check they do what we expect
|
|
68
70
|
for i in range(bands):
|
|
69
|
-
o =
|
|
71
|
+
o = yg.read_raster(target_path, band=i+1)
|
|
70
72
|
assert o.sum() == ((100 - (bands - 1)) * (100 - (bands - 1)) * (i + 1))
|