openforis-whisp 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openforis_whisp/__init__.py +68 -0
- openforis_whisp/data_conversion.py +348 -0
- openforis_whisp/datasets.py +695 -0
- openforis_whisp/logger.py +39 -0
- openforis_whisp/parameters/__init__.py +15 -0
- openforis_whisp/parameters/config_runtime.py +47 -0
- openforis_whisp/parameters/lookup_context_and_metadata.csv +13 -0
- openforis_whisp/parameters/lookup_gee_datasets.csv +155 -0
- openforis_whisp/pd_schemas.py +77 -0
- openforis_whisp/reformat.py +346 -0
- openforis_whisp/risk.py +329 -0
- openforis_whisp/stats.py +752 -0
- openforis_whisp/utils.py +154 -0
- openforis_whisp-0.0.1.dist-info/LICENSE +21 -0
- openforis_whisp-0.0.1.dist-info/METADATA +296 -0
- openforis_whisp-0.0.1.dist-info/RECORD +17 -0
- openforis_whisp-0.0.1.dist-info/WHEEL +4 -0
openforis_whisp/utils.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import ee
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
import importlib.resources as pkg_resources
|
|
8
|
+
|
|
9
|
+
from dotenv import load_dotenv
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from .logger import StdoutLogger
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
logger = StdoutLogger(__name__)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_example_data_path(filename):
|
|
19
|
+
"""
|
|
20
|
+
Get the path to an example data file included in the package.
|
|
21
|
+
|
|
22
|
+
Parameters:
|
|
23
|
+
-----------
|
|
24
|
+
filename : str
|
|
25
|
+
The name of the example data file.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
--------
|
|
29
|
+
str
|
|
30
|
+
The path to the example data file.
|
|
31
|
+
"""
|
|
32
|
+
return os.path.join("..", "tests", "fixtures", filename)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def load_env_vars() -> None:
|
|
36
|
+
"""Loads the environment variables required for testing the codebase.
|
|
37
|
+
|
|
38
|
+
Returns
|
|
39
|
+
-------
|
|
40
|
+
out : None
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
all_dotenv_paths = [Path(__file__).parents[2] / ".env", Path.cwd() / ".env"]
|
|
44
|
+
dotenv_loaded = False
|
|
45
|
+
|
|
46
|
+
for dotenv_path in all_dotenv_paths:
|
|
47
|
+
logger.logger.debug(f"dotenv_path: {dotenv_path}")
|
|
48
|
+
if dotenv_path.exists():
|
|
49
|
+
dotenv_loaded = load_dotenv(dotenv_path)
|
|
50
|
+
break
|
|
51
|
+
|
|
52
|
+
if not dotenv_loaded:
|
|
53
|
+
raise DotEnvNotFoundError
|
|
54
|
+
logger.logger.info(f"Loaded evironment variables from '{dotenv_path}'")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def init_ee() -> None:
|
|
58
|
+
"""Initialize earth engine according to the environment"""
|
|
59
|
+
|
|
60
|
+
# only do the initialization if the credential are missing
|
|
61
|
+
if not ee.data._credentials:
|
|
62
|
+
|
|
63
|
+
# if in test env use the private key
|
|
64
|
+
if "EE_PRIVATE_KEY" in os.environ:
|
|
65
|
+
|
|
66
|
+
# key need to be decoded in a file
|
|
67
|
+
content = base64.b64decode(os.environ["EE_PRIVATE_KEY"]).decode()
|
|
68
|
+
with open("ee_private_key.json", "w") as f:
|
|
69
|
+
f.write(content)
|
|
70
|
+
|
|
71
|
+
# connection to the service account
|
|
72
|
+
service_account = "test-sepal-ui@sepal-ui.iam.gserviceaccount.com"
|
|
73
|
+
credentials = ee.ServiceAccountCredentials(
|
|
74
|
+
service_account, "ee_private_key.json"
|
|
75
|
+
)
|
|
76
|
+
ee.Initialize(credentials)
|
|
77
|
+
logger.logger.info(f"Used env var")
|
|
78
|
+
|
|
79
|
+
# if in local env use the local user credential
|
|
80
|
+
else:
|
|
81
|
+
try:
|
|
82
|
+
load_env_vars()
|
|
83
|
+
logger.logger.info("Called 'ee.Initialize()'.")
|
|
84
|
+
ee.Initialize(project=os.environ["PROJECT"])
|
|
85
|
+
except ee.ee_exception.EEException:
|
|
86
|
+
logger.logger.info("Called 'ee.Authenticate()'.")
|
|
87
|
+
ee.Authenticate()
|
|
88
|
+
ee.Initialize(project=os.environ["PROJECT"])
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def clear_ee_credentials():
|
|
92
|
+
|
|
93
|
+
path_to_creds = Path().home() / ".config" / "earthengine" / "credentials"
|
|
94
|
+
if not path_to_creds.exists():
|
|
95
|
+
logger.logger.error(
|
|
96
|
+
f"GEE credentials file '{path_to_creds}' not found, could not de-authenticate."
|
|
97
|
+
)
|
|
98
|
+
else:
|
|
99
|
+
path_to_creds.unlink()
|
|
100
|
+
logger.logger.warning(f"GEE credentials file deleted.")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def remove_geometry_from_feature_collection(feature_collection):
|
|
104
|
+
"""Define the function to remove geometry from features in a feature collection"""
|
|
105
|
+
# Function to remove geometry from features
|
|
106
|
+
def remove_geometry(feature):
|
|
107
|
+
# Remove the geometry property
|
|
108
|
+
feature = feature.setGeometry(None)
|
|
109
|
+
return feature
|
|
110
|
+
|
|
111
|
+
# Apply the function to remove geometry to the feature collection
|
|
112
|
+
feature_collection_no_geometry = feature_collection.map(remove_geometry)
|
|
113
|
+
return feature_collection_no_geometry
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Compute centroids of each polygon
|
|
117
|
+
def get_centroid(feature, geo_id_column="Geo_id"):
|
|
118
|
+
keepProperties = [geo_id_column]
|
|
119
|
+
# Get the centroid of the feature's geometry.
|
|
120
|
+
centroid = feature.geometry().centroid(1)
|
|
121
|
+
# Return a new Feature, copying properties from the old Feature.
|
|
122
|
+
return ee.Feature(centroid).copyProperties(feature, keepProperties)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def buffer_point_to_required_area(feature, area, area_unit):
|
|
126
|
+
"""buffers feature to get a given area (needs math library); area unit in 'ha' or 'km2' (the default)"""
|
|
127
|
+
area = feature.get("REP_AREA")
|
|
128
|
+
|
|
129
|
+
# buffer_size = get_radius_m_to_buffer_for_given_area(area,"km2")# should work but untested in this function
|
|
130
|
+
|
|
131
|
+
buffer_size = (
|
|
132
|
+
(ee.Number(feature.get("REP_AREA")).divide(math.pi)).sqrt().multiply(1000)
|
|
133
|
+
) # calculating radius in metres from REP_AREA in km2
|
|
134
|
+
|
|
135
|
+
return ee.Feature(feature).buffer(buffer_size, 1)
|
|
136
|
+
### buffering (incl., max error parameter should be 0m. But put as 1m anyhow - doesn't seem to make too much of a difference for speed)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def get_radius_m_to_buffer_to_required_area(area, area_unit="km2"):
|
|
140
|
+
"""gets radius in metres to buffer to get an area (needs math library); area unit ha or km2 (the default)"""
|
|
141
|
+
if area_unit == "km2":
|
|
142
|
+
unit_fix_factor = 1000
|
|
143
|
+
elif area_unit == "ha":
|
|
144
|
+
unit_fix_factor = 100
|
|
145
|
+
radius = ee.Number(area).divide(math.pi).sqrt().multiply(unit_fix_factor)
|
|
146
|
+
return radius
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class DotEnvNotFoundError(FileNotFoundError):
|
|
150
|
+
def __init__(self) -> None:
|
|
151
|
+
super().__init__(
|
|
152
|
+
"Running tests requires setting an appropriate '.env' in the root directory or in your current working "
|
|
153
|
+
"directory. You may copy and edit the '.env.template' file from the root directory or from the README.",
|
|
154
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 lecrabe
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: openforis-whisp
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Whisp (What is in that plot) is an open-source solution which helps to produce relevant forest monitoring information and support compliance with deforestation-related regulations.
|
|
5
|
+
License: MIT
|
|
6
|
+
Keywords: whisp,geospatial,data-processing
|
|
7
|
+
Author: Andy Arnell
|
|
8
|
+
Author-email: and.arnell@fao.org
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Dist: country_converter (>=0.7,<2.0.0)
|
|
21
|
+
Requires-Dist: earthengine-api
|
|
22
|
+
Requires-Dist: geemap
|
|
23
|
+
Requires-Dist: geojson (>=2.5.0,<3.0.0)
|
|
24
|
+
Requires-Dist: ipykernel (>=6.17.1,<7.0.0)
|
|
25
|
+
Requires-Dist: numpy (>=1.21.0,<3.0.0)
|
|
26
|
+
Requires-Dist: pandas (>=1.3.0,<3.0.0)
|
|
27
|
+
Requires-Dist: pandera[io] (>=0.22.1,<1.0.0)
|
|
28
|
+
Requires-Dist: pydantic-core (>=2.14.0,<3.0.0)
|
|
29
|
+
Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
|
|
30
|
+
Requires-Dist: rsa (>=4.2,<5.0.0)
|
|
31
|
+
Requires-Dist: shapely (>=2.0.2,<3.0.0)
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
whisp
|
|
35
|
+
=====
|
|
36
|
+
[](https://github.com/openforis/sepal/blob/master/license.txt)
|
|
37
|
+
[](https://www.fao.org/contact-us/privacy-policy-applications-use/en)
|
|
38
|
+
[](https://doi.org/10.4060/cd0957en)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+

|
|
43
|
+
|
|
44
|
+
## Convergence of Evidence <a name="whisp_coe"></a>
|
|
45
|
+
***Whisp*** stands for "***Wh****at* ***is*** *in that* ***p****lot*"?
|
|
46
|
+
|
|
47
|
+
Numerous publicly available Earth Observation maps provide data on tree cover, land use, and forest disturbances. However, these maps often differ from one another because they use various definitions and classification systems. As a result, no single map can provide a complete picture of any specific area. To address this issue, the [Forest Data Partnership (FDaP)](https://www.forestdatapartnership.org) and the [AIM4Forests Programme](https://www.fao.org/in-action/aim4forests/en/) advocate for the **Convergence of Evidence** approach.
|
|
48
|
+
|
|
49
|
+
The Forest Data Partnership promotes this approach for forest and commodities monitoring, assuming that
|
|
50
|
+
- no single source of geospatial data can tell the whole story around any given plot of land;
|
|
51
|
+
- all the existing, published and available datasets contribute to telling that story.
|
|
52
|
+
<br><br>
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
## Contents
|
|
56
|
+
- [Whisp pathways](#whisp_pathways)
|
|
57
|
+
- [Whisp datasets](#whisp_datasets)
|
|
58
|
+
- [Whisp notebooks](#whisp_notebooks)
|
|
59
|
+
- [System setup](#whisp_setup)
|
|
60
|
+
- [Add data layers](#whisp_add_data)
|
|
61
|
+
- [Contribute to the code](#whisp_contribute)
|
|
62
|
+
- [Code of conduct](#whisp_conduct)
|
|
63
|
+
|
|
64
|
+
<br>
|
|
65
|
+
|
|
66
|
+
## Whisp pathways <a name="whisp_pathways"></a>
|
|
67
|
+
***Whisp*** can currently be used directly or implemented in your own code through three different pathways:
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
1. The Whisp App with its simple interface can be used [right here](https://whisp.openforis.org/) or called from other software by [API](https://whisp.openforis.org/documentation/api-guide). The Whisp App currently supports the processing of up to 500 geometries per job. The original JS & Python code behind the Whisp App and API can be found [here](https://github.com/forestdatapartnership/whisp-app).
|
|
71
|
+
|
|
72
|
+
2. [Whisp in Earthmap](https://whisp.earthmap.org/?aoi=WHISP&boundary=plot1&layers=%7B%22CocoaETH%22%3A%7B%22opacity%22%3A1%7D%2C%22JRCForestMask%22%3A%7B%22opacity%22%3A1%7D%2C%22planet_rgb%22%3A%7B%22opacity%22%3A1%2C%22date%22%3A%222020-12%22%7D%7D&map=%7B%22center%22%3A%7B%22lat%22%3A7%2C%22lng%22%3A4%7D%2C%22zoom%22%3A3%2C%22mapType%22%3A%22satellite%22%7D&statisticsOpen=true) supports the visualization of geometries on actual maps with the possibility to toggle different relevant map products around tree cover, commodities and deforestation. It is practical for demonstration purposes and spot checks of single geometries but not recommended for larger datasets.
|
|
73
|
+
|
|
74
|
+
3. Datasets of any size, especially when holding more than 500 geometries, can be "whisped" through the [Jupyter Notebook](whisp_feature_collection.ipynb) in this repository. They can either be uploaded as GeoJSONs or accessed through GEE assets. For the detailed procedure please go to the section [Whisp notebooks](#whisp_notebooks).
|
|
75
|
+
|
|
76
|
+
<br>
|
|
77
|
+
|
|
78
|
+
## Whisp datasets <a name="whisp_datasets"></a>
|
|
79
|
+
***Whisp*** implements the convergence of evidence approach by providing a transparent and public processing flow using datasets covering the following categories:
|
|
80
|
+
|
|
81
|
+
1) Tree and forest cover (at the end of 2020);
|
|
82
|
+
2) Commodities (i.e., crop plantations and other agricultural uses at the end of 2020);
|
|
83
|
+
3) Disturbances **before 2020** (i.e., degredation or deforestation until 2020-12-31);
|
|
84
|
+
4) Disturbances **after 2020** (i.e., degredation or deforestation from 2021-01-01 onward).
|
|
85
|
+
|
|
86
|
+
There are multiple datasets for each category. Find the full current [list of datasets used in Whisp here](https://github.com/forestdatapartnership/whisp/blob/main/layers_description.md).
|
|
87
|
+
Whisp checks the plots provided by the user by running zonal statistics on them to answer the following questions:
|
|
88
|
+
|
|
89
|
+
1) Was there tree cover in 2020?
|
|
90
|
+
2) Were there commodity plantations or other agricultural uses in 2020?
|
|
91
|
+
3) Were there disturbances until 2020-12-31?
|
|
92
|
+
4) Were there disturbances after 2020-12-31 / starting 2021-01-01?
|
|
93
|
+
|
|
94
|
+
If no treecover dataset indicates any tree cover for a plot by the end of 2020, **Whisp will deem the deforestation risk as low.**
|
|
95
|
+
|
|
96
|
+
If one or more treecover datasets indicate tree cover on a plot by the end of 2020, but a commodity dataset indicates agricultural use by the end of 2020, **Whisp will deem the deforestation risk as low.**
|
|
97
|
+
|
|
98
|
+
If treecover datasets indicate tree cover on a plot by late 2020, no commodity datasets indicate agricultural use, but a disturbance datasets indicates disturbances before the end of 2020, **Whisp will deem the deforestation risk as <u>low</u>.** Such deforestation has happened before the EUDR cutoff date and therefore does not count as high risk for the EUDR.
|
|
99
|
+
|
|
100
|
+
Now, if the datasets under 1., 2. & 3. indicate that there was tree cover, but no agriculture and no disturbances before or by the end of 2020, the Whisp algorithm checks whether degredation or deforestation have been reported in a disturbance dataset after 2020-12-31. If they have, **Whisp will deem the deforestation risk as <u>high</u>.** <br>
|
|
101
|
+
However, under the same circumstances but with <u>no</u> disturbances reported after 2020-12-31 there is insufficient evidence and the **Whisp output will be "More info needed".** Such can be the case for, e.g., cocoa or coffee grown under the shade of treecover or agroforestry.
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
*The Whisp algorithm visualized:*
|
|
105
|
+

|
|
106
|
+
|
|
107
|
+
--------------------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+

|
|
110
|
+
|
|
111
|
+
# Requirements
|
|
112
|
+
|
|
113
|
+
- A Google Earth Engine (GEE) account.
|
|
114
|
+
- A registered cloud GEE project.
|
|
115
|
+
- Some experience in Python or a similar language.
|
|
116
|
+
|
|
117
|
+
# Setup
|
|
118
|
+
Consider creating a virtual environment to keep your main python installation clean.
|
|
119
|
+
This can be done through this command line
|
|
120
|
+
```
|
|
121
|
+
python -m venv env
|
|
122
|
+
```
|
|
123
|
+
More info on virtual environments here: https://docs.python.org/3/library/venv.html
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
If installing from GitHUb
|
|
127
|
+
Clone the Whisp repo and install the package:
|
|
128
|
+
```
|
|
129
|
+
git clone https://github.com/forestdatapartnership/whisp.git
|
|
130
|
+
cd whisp/
|
|
131
|
+
pip install .
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Create a `.env` file with the required information for Earth Engine login (you may copy and edit the `.env.template`
|
|
135
|
+
file found in the root directory). The file must contain the following environment variables:
|
|
136
|
+
```
|
|
137
|
+
# Earth Engine project name
|
|
138
|
+
PROJECT="my-ee-project"
|
|
139
|
+
```
|
|
140
|
+
Where you must replace the GEE project name by your own.
|
|
141
|
+
|
|
142
|
+
Note: this should be a registered cloud project. If unsure of this check pic here: https://developers.google.com/earth-engine/cloud/assets
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# Contributing
|
|
146
|
+
|
|
147
|
+
Install the package in editable mode with the additional dependencies required for testing and running pre-commit hooks:
|
|
148
|
+
```
|
|
149
|
+
git clone https://github.com/forestdatapartnership/whisp.git
|
|
150
|
+
cd whisp/
|
|
151
|
+
pip install -e .[dev]
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Setup the pre-commit hooks:
|
|
155
|
+
```
|
|
156
|
+
pre-commit install
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
You should be able to run the Pytest suite by simple running the `pytest` command from the repo's root folder.
|
|
161
|
+
|
|
162
|
+
Please read the  for good practice recommendations.
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
# whisp pathways
|
|
166
|
+
***whisp*** can be implemented through at least three pathways
|
|
167
|
+
|
|
168
|
+
1. As an operationnal API in JS and Python
|
|
169
|
+
|
|
170
|
+
2. Through mobile applications to enable smallholder in the field directly generate and own the monitoring data associated with their plots of land
|
|
171
|
+
|
|
172
|
+
3. Integrated through Graphical User Interfaces for demonstration and verification purposes
|
|
173
|
+
|
|
174
|
+
4. Python Notebook implementation.
|
|
175
|
+
|
|
176
|
+
More info on Whisp can be found in [here](https://openknowledge.fao.org/items/e9284dc7-4b19-4f9c-b3e1-e6c142585865)
|
|
177
|
+
|
|
178
|
+
## How to Whisp Your Data Using the Python Notebook Pathway
|
|
179
|
+
|
|
180
|
+
The Python notebooks are currently set up to run in Sepal and to focus on polygon data provided by the user or geo_ids that allow the notebooks to access polygons stored and registered in the Asset Registry. NB: We value your feedback in terms of what input data we should support.
|
|
181
|
+
|
|
182
|
+
If your data is available as a feature collection and the GitHub repo is cloned, you are ready to start processing. We suggest first familiarizing yourself with running the notebooks using the default inputs in the notebook code. This will allow you to understand the expected outputs and the general functionality, as well as check if the setup worked successfully.
|
|
183
|
+
|
|
184
|
+
### Whisping a GeoJSON containing plot data
|
|
185
|
+
|
|
186
|
+
1. Open Jupyter Lab (see Apps).
|
|
187
|
+
2. Open the notebook `whisp_geojson.ipynb` from inside the "notebooks" folder.
|
|
188
|
+
3. To run the notebook cells, press the Play icon, or use the Shift + Enter shortcut. For more info on Jupyter and controlling notebooks, see [here](https://jupyter.org/try-jupyter/lab/).
|
|
189
|
+
4. Once you have initialised and agreed to the Google Earth Engine access requirements, you can process a geojson file containing plot informations.
|
|
190
|
+
5. The outputs from the notebook can be set to where you want to store them.
|
|
191
|
+
6. Outputs from the process include a CSV file `whisp_output_table.csv`.
|
|
192
|
+
7. NB If the processing job is large this may take some time. IF the outputs fail to process due to memory or time out errors, you can use the alternative function 'whisp_stats_geojson_to_drive', that sends a csv outpout to your Google drive. We are developing functionality to upload the csv automatically. In the meantime manually downloading from Google Drive and saving the csv is required. You can then import the csv as a pandas dataframe and use the 'validate_dataframe_using_lookups' function to get the Whisp output format (i.e., using a set schema for a standardised output, useful for showing in Power Bi etc).
|
|
193
|
+
|
|
194
|
+
### Adding risk indicators to your results
|
|
195
|
+
8. Finally, functions at the end of the notebook allow the user to add EUDR risk indicators to the table. NB: these risk indicators are still at the experimental stage.
|
|
196
|
+
|
|
197
|
+
### Output 1
|
|
198
|
+
|
|
199
|
+
- A CSV called `whisp_output_table.csv` contains results from the whisp processing.
|
|
200
|
+
|
|
201
|
+
### Output 2
|
|
202
|
+
|
|
203
|
+
- A CSV called `whisp_output_table_w_risk.csv` contains results from the whisp processing and risk indicators.
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
## Other Files
|
|
207
|
+
|
|
208
|
+
### Parameters Folder
|
|
209
|
+
|
|
210
|
+
A folder containing a series of Python scripts and a CSV. These files are used to define various parameters used in the analysis.
|
|
211
|
+
|
|
212
|
+
Key files include:
|
|
213
|
+
- `lookup_gee_datasets.csv` contains the list of input datasets, the order they will be displayed, which ones are to be excluded from the current analysis, and which ones are shown as flags (i.e., shown as presence or absence instead of figures for area/percentage coverage of the plot).
|
|
214
|
+
- `config_runtime.py` contains parameters such as default file and column names. **WARNING: USERS SHOULD NOT EDIT PACKAGE CONFIG ; IF THEY WANT TO, THEY SHOULD INSTALL THE PACKAGE IN EDITABLE MODE IN ORDER TO BE ABLE TO MODIFY CONFIGS WITHOUT HAVING TO RE-INSTALL THE PACKGE EVERY TIME**
|
|
215
|
+
|
|
216
|
+
### src code
|
|
217
|
+
|
|
218
|
+
Various functions for the main Whisp analysis along with some for interacting with the AgStack Asset Registry.
|
|
219
|
+
|
|
220
|
+
Key files:
|
|
221
|
+
- `datasets.py` contains a series of functions related to the creation of a single multiband GEE image to be used in the Whisp summary statistics analysis.
|
|
222
|
+
- `stats.py` contains functions to run the Whisp analysis for each of the various datasets and to provide results for coverage of each plot as a percentage (or as an area in hectares).
|
|
223
|
+
|
|
224
|
+
## Setting Up Your System
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
### Setting Up SEPAL
|
|
228
|
+
|
|
229
|
+
SEPAL is closely linked to Google Earth Engine (GEE), a Google-powered Earth-observation cloud-computing platform, as it builds in many of its functionalities. Currently, you will need to have connected SEPAL and GEE accounts. SEPAL provides a stable processing environment and allows you to link up with your Google account saving time with permissions. Currently, we are supporting the use within SEPAL, but you can run the scripts outside of SEPAL if required, although this will require extra code to account for the various dependencies SEPAL already has built in.
|
|
230
|
+
|
|
231
|
+
1. Login to SEPAL.
|
|
232
|
+
2. Start an instance (see Terminal info) to provide you with free processing power.
|
|
233
|
+
3. If you don’t have SEPAL set up:
|
|
234
|
+
- To create a SEPAL account, please follow the registration steps described [here](https://docs.sepal.io/en/latest/setup/register.html) and then familiarize yourself with the tool by exploring its interface.
|
|
235
|
+
- To create a Google Earth Engine (GEE) account, please follow these steps and don’t forget to initialize the home folder.
|
|
236
|
+
|
|
237
|
+
### Setting Up the Whisp GitHub Repository
|
|
238
|
+
|
|
239
|
+
1. Make sure you have a GitHub account set up.
|
|
240
|
+
2. To run the Whisp notebooks in SEPAL, you need to copy the Whisp repository into your SEPAL files. The notebooks rely on functions and parameters that are stored in other files in this repository.
|
|
241
|
+
3. To clone (i.e., copy) the GitHub repository so that it is accessible in SEPAL, type into the SEPAL terminal:
|
|
242
|
+
|
|
243
|
+
```sh
|
|
244
|
+
git clone https://github.com/forestdatapartnership/whisp.git
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
4. If this works, on the left-hand pane you should now be able to view a Whisp folder containing Notebooks along with other supporting files and folders.
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
## How to add data layers to Whisp
|
|
251
|
+
|
|
252
|
+
There are two main approaches: to request a layer be incorporated into the core Whisp inputs, or to add in your own data directly to complement the core ones in Whisp
|
|
253
|
+
|
|
254
|
+
### Requesting a dataset addition
|
|
255
|
+
If you think a particular dataset has wide applicability for Whisp users, you can request it be added to the main Whisp repository by logging as an issue in Github [here] (https://github.com/forestdatapartnership/whisp/issues/).
|
|
256
|
+
|
|
257
|
+
### Adding your own dataset for a bespoke Whisp analysis (using the Python Notebooks)
|
|
258
|
+
Adding your To add other datasets, such as that are specific to your circumstances, or can’t be shared directly in GEE, follow the steps and guidance below.
|
|
259
|
+
|
|
260
|
+
1) Edit modules/datasets.py and add in a function to make a binary GEE image (i.e., where values are either 0 or 1*). Make sure the function name ends with "_prep", as only functions with this suffix are used.
|
|
261
|
+
2) Choose a name to represent the data layer in the final CSV output. Place in speech brackets in the .rename() section at the end of the function. See examples elsewhere in the functions in this script.
|
|
262
|
+
3) Edit parameters/lookup_gee_datasets.csv to include the chosen dataset name in a new row. Make sure other relevant columns are filled in.
|
|
263
|
+
|
|
264
|
+
The above assumes a single band image that is being included, which results in a single column being added.
|
|
265
|
+
If you have multiband images to add and want each band to be a layer in Whisp, make sure each band is named.
|
|
266
|
+
Make sure to add all the bands to the lookup CSV (see Step 3), else they won’t appear in the output.
|
|
267
|
+
|
|
268
|
+
How to fill out the columns parameters/lookup_gee_datasets.csv
|
|
269
|
+
a. name: the name for the dataset column. NB must be exactly the same as the name of the image band in step 1.
|
|
270
|
+
b. order: choose a number for where you want the dataset column placed in the CSV output.
|
|
271
|
+
c. theme: a word denoting the dataset Theme. Currently there are five themes where i to iv correspond to:
|
|
272
|
+
i. treecover: for forest or treecover at the end of 2020
|
|
273
|
+
ii. commodities: representing commodities in 2020 (typically ones that tree cover might be confused with in remote sensing products).
|
|
274
|
+
iii. disturbance_before: forest/tree cover disturbance before the end of 2020
|
|
275
|
+
iv. disturbance_after: forest/tree cover disturbance after the end of 2020
|
|
276
|
+
v. ancillary: other relevant layers, such as representing protected areas or areas of importance for biodiversity.
|
|
277
|
+
d. use_for_risk: if 1 is added here this dataset is included in the risk calculations. The type of risk indicator it will contribute to is automatically governed by the “theme” column.
|
|
278
|
+
NB if there is in a 1 in the "exclude_from_output" column this over-rules all of the above and the dataset is ignored. There are functions (in the modules/risk.py), to create lists for each of the 4 indicators from the lookup csv. These are used in the "whisp_risk" function for creating default columns to include in the final overall risk column.
|
|
279
|
+
e. exclude_from_output: removes the column from the formatted final table (to remove input code out the function in the datasets.py)
|
|
280
|
+
f. col_type:
|
|
281
|
+
- choose 'float32' (most )
|
|
282
|
+
- exceptions are 'bool' for showing True/False, where values >0 gives True.
|
|
283
|
+
e. is nullable: set to 1
|
|
284
|
+
f. is required: set to 0
|
|
285
|
+
g. corresponding variable: the name of the function for creating the dataset in datasets.py (should end with "_prep")
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
### Tips for preparing and adding in your data
|
|
289
|
+
• It’s sometimes easier to do initial checks in JavaScript and check all looks ok on the map in Code Editor, and then convert the code into Python. Tools that can help convert include AI interfaces such as ChatGPT, or [geemap] (https://giswqs.medium.com/15-converting-earth-engine-javascripts-to-python-code-with-just-a-few-mouse-clicks-6aa02b1268e1/).
|
|
290
|
+
• Check your data: Python functions will still need sense checking and putting on a map is one way to do this using functions in [geemap] (https://geemap.org/notebooks/09_plotting/)
|
|
291
|
+
• A binary input image is expected, but non-integer values are allowed if they range between 0 and 1. This is most appropriate for datasets that have proportion of coverage in a pixel (e.g., a value of 0.5 would represent having half the pixel covered).
|
|
292
|
+
• If you are adding timeseries data, when creating the function you can use loops/mapping to compile a multiband input and to name each band accordingly.
|
|
293
|
+
|
|
294
|
+
## Contributing to the Whisp code base
|
|
295
|
+
Contributions to the Whisp code in GitHub are welcome. They can be made by forking the repository making and pushing the required changes, then making a pull request to the Whisp repository. After briefly reviewing the request, we can make a branch for which to make a new pull request to. If in doubt get in contact first or log as an issue [here] (https://github.com/forestdatapartnership/whisp/issues/).
|
|
296
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
openforis_whisp/__init__.py,sha256=_7bgVQ8q9X9WeFpnzqN0uSD-TgM99B6G2DPWXROtf78,2301
|
|
2
|
+
openforis_whisp/data_conversion.py,sha256=96en23_ysP7OfOJNGBssDyaX8seuzYGUarqdgnbI4SU,10961
|
|
3
|
+
openforis_whisp/datasets.py,sha256=J-gThvw_4ZA6HyQXFHY2C1hHUzPN4tX-Ap_EbL3c-N0,23322
|
|
4
|
+
openforis_whisp/logger.py,sha256=bcKu82a3njNIBiRghDcO_cC-SOK62cYAYuTaSWq7s9I,1354
|
|
5
|
+
openforis_whisp/parameters/__init__.py,sha256=KL7iORJVjSpZatYjoyWckcmQJnE89_DBC8R6_0_eR6o,349
|
|
6
|
+
openforis_whisp/parameters/config_runtime.py,sha256=gsVOTpcf5oKMJUeoN5A1klCvC52eA4fotD9mGUPgi8w,1140
|
|
7
|
+
openforis_whisp/parameters/lookup_context_and_metadata.csv,sha256=4eciReFRPRSDeKBiUiODbhFrtZcguxgxc2DOlMB6qG8,1003
|
|
8
|
+
openforis_whisp/parameters/lookup_gee_datasets.csv,sha256=nH_StTYmxY74lYBsWNbK60FhUBmGzXiVyWdJ-sJ9ezA,11627
|
|
9
|
+
openforis_whisp/pd_schemas.py,sha256=W_ocS773LHfc05dJqvWRa-bRdX0wKFoNp0lMxgFx94Y,2681
|
|
10
|
+
openforis_whisp/reformat.py,sha256=_ByLR5bu_gPecH6W4jxymkcIN5Hph3bRIuRfjZByNBU,12125
|
|
11
|
+
openforis_whisp/risk.py,sha256=qgrBnioYS_vMPWs_rXFR5k_TacxBACHbeYmPhBmqMrM,13367
|
|
12
|
+
openforis_whisp/stats.py,sha256=Y-IfazrTA_O5wQ8YE0GypXQYopp1V-ym_jPLwtNCw4M,26405
|
|
13
|
+
openforis_whisp/utils.py,sha256=hpeY9aA3BND2m9c15PZ6_nClemsfiVNUEzA4pQXfztA,5330
|
|
14
|
+
openforis_whisp-0.0.1.dist-info/LICENSE,sha256=nqyqICO95iw_iwzP1t_IIAf7ZX3DPbL_M9WyQfh2q1k,1085
|
|
15
|
+
openforis_whisp-0.0.1.dist-info/METADATA,sha256=jgTZka6wdCS20shNgJ9QFS2w-szv0o304cpRTZ217cQ,19759
|
|
16
|
+
openforis_whisp-0.0.1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
|
|
17
|
+
openforis_whisp-0.0.1.dist-info/RECORD,,
|