wofs 1.6.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wofs-1.6.6/.github/workflows/push.yaml +66 -0
- wofs-1.6.6/.gitignore +100 -0
- wofs-1.6.6/.yamllint +22 -0
- wofs-1.6.6/PKG-INFO +184 -0
- wofs-1.6.6/README.rst +154 -0
- wofs-1.6.6/check-code.sh +9 -0
- wofs-1.6.6/config/wofs_albers.yaml +113 -0
- wofs-1.6.6/config/wofs_modified_albers.yaml +114 -0
- wofs-1.6.6/example.ipynb +131 -0
- wofs-1.6.6/pylintrc +241 -0
- wofs-1.6.6/pyproject.toml +6 -0
- wofs-1.6.6/runtests-docker.sh +7 -0
- wofs-1.6.6/setup.cfg +13 -0
- wofs-1.6.6/setup.py +50 -0
- wofs-1.6.6/tests/Test core algorithm.ipynb +481 -0
- wofs-1.6.6/tests/create_test_input_data.py +39 -0
- wofs-1.6.6/tests/sample_c3_sr.nc +0 -0
- wofs-1.6.6/tests/sample_wofl.nc +0 -0
- wofs-1.6.6/tests/test_bitmask_values.py +60 -0
- wofs-1.6.6/tests/test_terrain.py +39 -0
- wofs-1.6.6/tests/test_virtualproduct.py +62 -0
- wofs-1.6.6/tests/test_wofs.py +27 -0
- wofs-1.6.6/wofs/__init__.py +5 -0
- wofs-1.6.6/wofs/_version.py +16 -0
- wofs-1.6.6/wofs/boilerplate.py +11 -0
- wofs-1.6.6/wofs/classifier.py +232 -0
- wofs-1.6.6/wofs/constants.py +49 -0
- wofs-1.6.6/wofs/filters.py +167 -0
- wofs-1.6.6/wofs/terrain.py +150 -0
- wofs-1.6.6/wofs/virtualproduct.py +138 -0
- wofs-1.6.6/wofs/wofls.py +109 -0
- wofs-1.6.6/wofs/wofs_app.py +730 -0
- wofs-1.6.6/wofs-summary/confidence.ipynb +757 -0
- wofs-1.6.6/wofs-summary/confidence.py +341 -0
- wofs-1.6.6/wofs-summary/job.sh +20 -0
- wofs-1.6.6/wofs-summary/simple.py +203 -0
- wofs-1.6.6/wofs.egg-info/PKG-INFO +184 -0
- wofs-1.6.6/wofs.egg-info/SOURCES.txt +41 -0
- wofs-1.6.6/wofs.egg-info/dependency_links.txt +1 -0
- wofs-1.6.6/wofs.egg-info/entry_points.txt +2 -0
- wofs-1.6.6/wofs.egg-info/requires.txt +13 -0
- wofs-1.6.6/wofs.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
name: Test and Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
test:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
strategy:
|
|
10
|
+
matrix:
|
|
11
|
+
python-version: [3.8, 3.9]
|
|
12
|
+
name: Python ${{ matrix.python-version }}
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v2
|
|
16
|
+
with:
|
|
17
|
+
fetch-depth: 0
|
|
18
|
+
|
|
19
|
+
- name: Install dependencies
|
|
20
|
+
run: |
|
|
21
|
+
sudo apt-get update && sudo apt-get install -f libudunits2-dev
|
|
22
|
+
pip install .[test]
|
|
23
|
+
|
|
24
|
+
- name: Run tests
|
|
25
|
+
run: ./check-code.sh
|
|
26
|
+
|
|
27
|
+
- name: Build package
|
|
28
|
+
run: |
|
|
29
|
+
python setup.py sdist bdist_wheel
|
|
30
|
+
|
|
31
|
+
- name: Upload artifacts
|
|
32
|
+
uses: actions/upload-artifact@v2
|
|
33
|
+
with:
|
|
34
|
+
name: packages
|
|
35
|
+
path: dist
|
|
36
|
+
|
|
37
|
+
- uses: codecov/codecov-action@v1
|
|
38
|
+
with:
|
|
39
|
+
env_vars: OS,PYTHON
|
|
40
|
+
file: ./coverage.xml
|
|
41
|
+
|
|
42
|
+
deploy-packages:
|
|
43
|
+
if: startsWith(github.ref, 'refs/tags/')
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
needs: test
|
|
46
|
+
steps:
|
|
47
|
+
- name: Download a single artifact
|
|
48
|
+
uses: actions/download-artifact@v2
|
|
49
|
+
with:
|
|
50
|
+
name: packages
|
|
51
|
+
path: dist
|
|
52
|
+
|
|
53
|
+
- name: Display directory structure of downloaded files
|
|
54
|
+
run: ls -lR
|
|
55
|
+
|
|
56
|
+
- name: Deploy packages
|
|
57
|
+
uses: jakejarvis/s3-sync-action@master
|
|
58
|
+
with:
|
|
59
|
+
args: --acl public-read --follow-symlinks
|
|
60
|
+
env:
|
|
61
|
+
AWS_S3_BUCKET: "datacube-core-deployment"
|
|
62
|
+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
63
|
+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
64
|
+
AWS_REGION: "ap-southeast-2" # optional: defaults to us-east-1
|
|
65
|
+
SOURCE_DIR: "dist" # optional: defaults to entire repository
|
|
66
|
+
DEST_DIR: "wofs"
|
wofs-1.6.6/.gitignore
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
|
|
5
|
+
# C extensions
|
|
6
|
+
*.so
|
|
7
|
+
|
|
8
|
+
# Distribution / packaging
|
|
9
|
+
.Python
|
|
10
|
+
env/
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
*.egg-info/
|
|
23
|
+
.installed.cfg
|
|
24
|
+
*.egg
|
|
25
|
+
|
|
26
|
+
# PyInstaller
|
|
27
|
+
# Usually these files are written by a python script from a template
|
|
28
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
29
|
+
*.manifest
|
|
30
|
+
*.spec
|
|
31
|
+
|
|
32
|
+
# Installer logs
|
|
33
|
+
pip-log.txt
|
|
34
|
+
pip-delete-this-directory.txt
|
|
35
|
+
|
|
36
|
+
# Unit test / coverage reports
|
|
37
|
+
htmlcov/
|
|
38
|
+
.tox/
|
|
39
|
+
.coverage
|
|
40
|
+
.coverage.*
|
|
41
|
+
.cache
|
|
42
|
+
nosetests.xml
|
|
43
|
+
coverage.xml
|
|
44
|
+
*,cover
|
|
45
|
+
.hypothesis
|
|
46
|
+
|
|
47
|
+
# Translations
|
|
48
|
+
*.mo
|
|
49
|
+
*.pot
|
|
50
|
+
|
|
51
|
+
# Django stuff:
|
|
52
|
+
*.log
|
|
53
|
+
|
|
54
|
+
# Sphinx documentation
|
|
55
|
+
docs/_build/
|
|
56
|
+
|
|
57
|
+
# PyBuilder
|
|
58
|
+
target/
|
|
59
|
+
.idea/
|
|
60
|
+
|
|
61
|
+
# iPython Notebook
|
|
62
|
+
.ipynb_checkpoints
|
|
63
|
+
|
|
64
|
+
# PBS Job manager output
|
|
65
|
+
*.[eo][0-9]*
|
|
66
|
+
log/*
|
|
67
|
+
!log/README.md
|
|
68
|
+
|
|
69
|
+
# images
|
|
70
|
+
*.tif
|
|
71
|
+
*.tiff
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# pdf files
|
|
75
|
+
*.pdf
|
|
76
|
+
|
|
77
|
+
#LaTeX files
|
|
78
|
+
#
|
|
79
|
+
*.log
|
|
80
|
+
*.out
|
|
81
|
+
*.ps
|
|
82
|
+
*.aux
|
|
83
|
+
|
|
84
|
+
# archive files
|
|
85
|
+
*.tar
|
|
86
|
+
*.tar.gz
|
|
87
|
+
*.zip
|
|
88
|
+
|
|
89
|
+
# private, local files
|
|
90
|
+
local*
|
|
91
|
+
zlocal*
|
|
92
|
+
private*
|
|
93
|
+
scratch/*
|
|
94
|
+
|
|
95
|
+
.pytest_cache/
|
|
96
|
+
.vscode
|
|
97
|
+
|
|
98
|
+
wofs/_version.py
|
|
99
|
+
.benchmarks/
|
|
100
|
+
tests/.benchmarks/
|
wofs-1.6.6/.yamllint
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
extends: default
|
|
2
|
+
|
|
3
|
+
rules:
|
|
4
|
+
# We could enable these if we find a decent yaml autoformatter.
|
|
5
|
+
# (yamlfmt in pip is too buggy at the moment for our files)
|
|
6
|
+
#
|
|
7
|
+
# The effort of fixing them by hand, especially auto-generated yamls, is not
|
|
8
|
+
# currently worth it.
|
|
9
|
+
|
|
10
|
+
# Many tools (eg. generated secure keys) don't output wrapped lines.
|
|
11
|
+
line-length: disable
|
|
12
|
+
|
|
13
|
+
# The existing documents will need to be cleaned up first
|
|
14
|
+
indentation: disable
|
|
15
|
+
|
|
16
|
+
# Pedantry & existing docs don't have it.
|
|
17
|
+
document-start: disable
|
|
18
|
+
|
|
19
|
+
# Warning that truthy values are not quoted.
|
|
20
|
+
# All documents currently use "True" without quotes, so this would be a
|
|
21
|
+
# larger change across almost every doc.
|
|
22
|
+
truthy: disable
|
wofs-1.6.6/PKG-INFO
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: wofs
|
|
3
|
+
Version: 1.6.6
|
|
4
|
+
Summary: Water Observations from Space - Digital Earth Australia
|
|
5
|
+
Home-page: https://github.com/GeoscienceAustralia/wofs
|
|
6
|
+
Author: Geoscience Australia
|
|
7
|
+
Maintainer: Geoscience Australia
|
|
8
|
+
Maintainer-email:
|
|
9
|
+
License: Apache License 2.0
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.6
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
18
|
+
Requires-Dist: datacube
|
|
19
|
+
Requires-Dist: scipy
|
|
20
|
+
Requires-Dist: ephem
|
|
21
|
+
Requires-Dist: xarray>=0.14.1
|
|
22
|
+
Provides-Extra: test
|
|
23
|
+
Requires-Dist: pytest; extra == "test"
|
|
24
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
25
|
+
Requires-Dist: mock; extra == "test"
|
|
26
|
+
Requires-Dist: pylint; extra == "test"
|
|
27
|
+
Requires-Dist: hypothesis; extra == "test"
|
|
28
|
+
Requires-Dist: compliance-checker; extra == "test"
|
|
29
|
+
Requires-Dist: black; extra == "test"
|
|
30
|
+
|
|
31
|
+
Water Observation from Space (WOfS)
|
|
32
|
+
====================================
|
|
33
|
+
|
|
34
|
+
:Disclaimer:
|
|
35
|
+
This repository is *in development*, use at your own risk.
|
|
36
|
+
|
|
37
|
+
:License:
|
|
38
|
+
The Apache 2.0 license applies to this open source code.
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
This WOfS version applies the original published WOfS decision-tree algorithm,
|
|
42
|
+
but is updated to use the Open Data Cube and
|
|
43
|
+
`xarray <http://xarray.pydata.org/en/stable/>`_ for data access.
|
|
44
|
+
|
|
45
|
+
Specifically, this version also decouples the production of water extent tiles
|
|
46
|
+
from the production of the statistical summary mosaics, and is intended to
|
|
47
|
+
improve consistency with other datacube applications (e.g., parallelisation
|
|
48
|
+
of the workflow employs *distributed* rather than *luigi*).
|
|
49
|
+
|
|
50
|
+
Codebase outline
|
|
51
|
+
----------------
|
|
52
|
+
|
|
53
|
+
The water-specific code (as distinct from packaging boilerplate) is located
|
|
54
|
+
in the "wofs" directory, other than metadata that may be located in the the
|
|
55
|
+
config yaml.
|
|
56
|
+
|
|
57
|
+
Installation
|
|
58
|
+
============
|
|
59
|
+
|
|
60
|
+
The WOfS package can be installed by running:
|
|
61
|
+
|
|
62
|
+
pip install --index-url https://packages.dea.ga.gov.au/ wofs
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
For Digital Earth Australia Users
|
|
66
|
+
---------------------------------
|
|
67
|
+
|
|
68
|
+
WOfS is available as a part of Digital Earth Australia environment modules on the NCI. These can be used
|
|
69
|
+
after logging into the NCI by running:
|
|
70
|
+
|
|
71
|
+
module load dea
|
|
72
|
+
|
|
73
|
+
Algorithm
|
|
74
|
+
=========
|
|
75
|
+
|
|
76
|
+
WOFLs
|
|
77
|
+
-----
|
|
78
|
+
|
|
79
|
+
Water Observation Feature Layers are the temporal foliation of water extents.
|
|
80
|
+
These consist of an 8-bit integer raster band.
|
|
81
|
+
|
|
82
|
+
- **Decision tree:** The standard classifier is *band maths* performed on 6 EO source bands (TM 1-5, 7). A published tree with 21 nodes, producing boolean output, comprising of thresholds applied to three raw bands (TM 1, 3, 7) and three band-pair ratio-indices (NDI 52, 43, 72).
|
|
83
|
+
- **Filter masks:** various flags are accumulated onto the output band. Inputs are the landsat image, the pixel quality product, and the elevation model. (The difficulty is generating some of the flags, e.g. terrain shadow.)
|
|
84
|
+
|
|
85
|
+
Summary
|
|
86
|
+
-------
|
|
87
|
+
|
|
88
|
+
The summary product has multiple parts:
|
|
89
|
+
|
|
90
|
+
- a mean mosaic of the wofls (i.e. fraction of clear observations that are wet);
|
|
91
|
+
- a confidence estimate. This is a logistic function wrapping a linear combination (with published weights) of several inputs: 0. mean mosaic of the wofls, 1. multi-res valley bottom flatness, 2. MODIS open water likelihood, hydrological geofabric, 3. slope, 4-12. hydrological geofabric (boolean vectors), 13. Aus Stat Geog Standard (urban boolean).
|
|
92
|
+
- Filtered summary, i.e., mean mosaic clipped to always-dry where confidence is below a threshold. (Would also be interesting to see confidence applied as an opacity alpha channel to the mean mosaic?)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
Notes and ideas
|
|
96
|
+
===============
|
|
97
|
+
|
|
98
|
+
Profiling
|
|
99
|
+
---------
|
|
100
|
+
|
|
101
|
+
Implementation of potential optimisations was deliberately deferred, until memory, CPU and IO profiling could take place.
|
|
102
|
+
|
|
103
|
+
Results (below) indicate that memory is already within the 2GB/core available, that IO is not a significant bottleneck (before scaling), and that speed is unlikely to improve dramatically (since limited by intrinsically demanding aspects of the current terrain-shadow algorithm); therefore significant optimisation effort may not be warranted.
|
|
104
|
+
|
|
105
|
+
19/9/02016: Querying test cube and writing 4 tiles (16MB each). /usr/bin/time showed 1min30sec walltime, ~135% CPU usage, ~10% system (rather than user time), ~1.5GB max resident. cProfile time graph indicated 2.4% spent on imports, 4% on database queries, 26% on grid workflow (including >5% on rasterio read and 19% on rasterio reproject) and 65% on the core algorithm. The latter is dominated by the terrain filtering (56%, alongside 4.4% decision tree, 3.3% PQ, 1.1% EO filter). It includes 1.9% on the Sobel operation, 9.5% row shading (of which 8% is python code, as is 5.8% of shadows and slope), and 37% rotating the image (scipy). Dilation also totals 4%. Summary:
|
|
106
|
+
|
|
107
|
+
- 20% potential speedup by storing DSM in the same projection as EO, or by orchestrating execution to avoid reloading DSM redundantly.
|
|
108
|
+
- Most of the time is spent on terrain, but only 5-10% speedup plausible by better implementation.
|
|
109
|
+
- Most limiting factor is rotating the DSM (to approximately align with sunlight) but nontrivial to improve or mitigate this. (May or may not be amenable to cheaper interpolation methods or an algorithm that traverses the array differently.)
|
|
110
|
+
|
|
111
|
+
Overlaps
|
|
112
|
+
--------
|
|
113
|
+
The Landsats collect a swath of data as they pass over the continent.
|
|
114
|
+
Traditionally, each pass is segmented into overlapping scenes for processing
|
|
115
|
+
separately. This necessitates measures to avoid double-counting duplicated
|
|
116
|
+
observations. Potential alternate measures would include:
|
|
117
|
+
|
|
118
|
+
- Whole pass based processing. (Upstream software not yet available.)
|
|
119
|
+
- EO archive of reconstituted seamless passes,
|
|
120
|
+
e.g., fusing scene-overlaps during ingestion.
|
|
121
|
+
(Renounced in current datacube iteration.)
|
|
122
|
+
- Duplicate-free water extents, e.g., fusing the inputs to wofl generation.
|
|
123
|
+
(Interferes with potential use of scene middle pixel timestamp as a primary
|
|
124
|
+
key for matching EO scenes to wofls.)
|
|
125
|
+
- Downstream each user/application perform wofl fusing.
|
|
126
|
+
(Beyond capability of the generic API; requires sharing wofs-specific code.)
|
|
127
|
+
|
|
128
|
+
The usefulness of preserving observation-duplicates (e.g. for investigation of
|
|
129
|
+
sensitivity to uncontrolled upstream processing parameters) is narrow.
|
|
130
|
+
|
|
131
|
+
Classifier
|
|
132
|
+
----------
|
|
133
|
+
|
|
134
|
+
It may improve performance and readability to represent the decision tree as a numexpr statement (nested across multiple lines). This could additionally include some of the mask logic.
|
|
135
|
+
|
|
136
|
+
Ideally the PQ product might be a band in the EO product (and include terrain related bitflags).
|
|
137
|
+
|
|
138
|
+
Alternative algorithms are under development elsewhere.
|
|
139
|
+
|
|
140
|
+
Terrain
|
|
141
|
+
-------
|
|
142
|
+
|
|
143
|
+
Terrain algorithms usually begin with finding the gradient component along each of the two axes, typically by operating with a 3x3 kernel. One example is the Rook's case (simply using nearest neighbours on either side of the pixel, which turns out to be a 2nd order finite difference method). Another is the Sobel operator, which additionally applies smoothing along the orthogonal axis. Tang and Pilesjo 2011 showed these belong to a variety of methods which produce statistically similar results (different from a more naive and unbalanced method of differencing the central cell with one neighbour along each axis). Jones 1998 found the Rook's case to give the best accuracy (narrowly followed by Sobel), but the methodology (e.g. noise-free synthetic) may have been biased (to favour balanced methods with more compact footprints). Zhou and Liu 2004 added noise to a synthetic, confirming the Rook's case to be optimal in absence of noise but the Sobel operator was more robust to the noise.
|
|
144
|
+
|
|
145
|
+
Clouds
|
|
146
|
+
------
|
|
147
|
+
|
|
148
|
+
Currently, cloud and cloud shadow are detected per scene, which is suboptimal at contiguous boundaries.
|
|
149
|
+
|
|
150
|
+
Improved masking algorithms are anticipated, e.g. as median mosaics become available, or possibly incorporating weather data.
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
Packaging and Releases
|
|
154
|
+
======================
|
|
155
|
+
|
|
156
|
+
Versioning
|
|
157
|
+
----------
|
|
158
|
+
|
|
159
|
+
The version number is based on the **algorithm version
|
|
160
|
+
number**, which at the moment stands at 1.4. See the `CMI Record for the WOfS Algorithm
|
|
161
|
+
<http://cmi.ga.gov.au/node/166>`_.
|
|
162
|
+
|
|
163
|
+
For minor code changes not affecting the algorithm, increment the least significant digit of the version number.
|
|
164
|
+
|
|
165
|
+
Releases
|
|
166
|
+
--------
|
|
167
|
+
|
|
168
|
+
To release a new package of WOfS, `create a new release <https://github.com/GeoscienceAustralia/wofs/releases/new>`_
|
|
169
|
+
using GitHub, with a suitably tagged version number.
|
|
170
|
+
|
|
171
|
+
The Continuous Integration service will run tests, create source and binary distribution packages, and upload them
|
|
172
|
+
to https://packages.dea.ga.gov.au/.
|
|
173
|
+
|
|
174
|
+
To build a new package for WOfS, Then, from the base directory of the project run:
|
|
175
|
+
|
|
176
|
+
python setup.py sdist bdist_wheel
|
|
177
|
+
|
|
178
|
+
This will create a ``source distribution`` and a ``binary wheel`` distribution in the ``dist/`` directory.
|
|
179
|
+
|
|
180
|
+
To have the package included in the **DEA Environment Module** upload it to s3://datacube-core-deployment/wofs/ by
|
|
181
|
+
running:
|
|
182
|
+
|
|
183
|
+
aws s3 cp dist/ s3://datacube-core-deployment/wofs/ --recursive
|
|
184
|
+
|
wofs-1.6.6/README.rst
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
Water Observation from Space (WOfS)
|
|
2
|
+
====================================
|
|
3
|
+
|
|
4
|
+
:Disclaimer:
|
|
5
|
+
This repository is *in development*, use at your own risk.
|
|
6
|
+
|
|
7
|
+
:License:
|
|
8
|
+
The Apache 2.0 license applies to this open source code.
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
This WOfS version applies the original published WOfS decision-tree algorithm,
|
|
12
|
+
but is updated to use the Open Data Cube and
|
|
13
|
+
`xarray <http://xarray.pydata.org/en/stable/>`_ for data access.
|
|
14
|
+
|
|
15
|
+
Specifically, this version also decouples the production of water extent tiles
|
|
16
|
+
from the production of the statistical summary mosaics, and is intended to
|
|
17
|
+
improve consistency with other datacube applications (e.g., parallelisation
|
|
18
|
+
of the workflow employs *distributed* rather than *luigi*).
|
|
19
|
+
|
|
20
|
+
Codebase outline
|
|
21
|
+
----------------
|
|
22
|
+
|
|
23
|
+
The water-specific code (as distinct from packaging boilerplate) is located
|
|
24
|
+
in the "wofs" directory, other than metadata that may be located in the the
|
|
25
|
+
config yaml.
|
|
26
|
+
|
|
27
|
+
Installation
|
|
28
|
+
============
|
|
29
|
+
|
|
30
|
+
The WOfS package can be installed by running:
|
|
31
|
+
|
|
32
|
+
pip install --index-url https://packages.dea.ga.gov.au/ wofs
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
For Digital Earth Australia Users
|
|
36
|
+
---------------------------------
|
|
37
|
+
|
|
38
|
+
WOfS is available as a part of Digital Earth Australia environment modules on the NCI. These can be used
|
|
39
|
+
after logging into the NCI by running:
|
|
40
|
+
|
|
41
|
+
module load dea
|
|
42
|
+
|
|
43
|
+
Algorithm
|
|
44
|
+
=========
|
|
45
|
+
|
|
46
|
+
WOFLs
|
|
47
|
+
-----
|
|
48
|
+
|
|
49
|
+
Water Observation Feature Layers are the temporal foliation of water extents.
|
|
50
|
+
These consist of an 8-bit integer raster band.
|
|
51
|
+
|
|
52
|
+
- **Decision tree:** The standard classifier is *band maths* performed on 6 EO source bands (TM 1-5, 7). A published tree with 21 nodes, producing boolean output, comprising of thresholds applied to three raw bands (TM 1, 3, 7) and three band-pair ratio-indices (NDI 52, 43, 72).
|
|
53
|
+
- **Filter masks:** various flags are accumulated onto the output band. Inputs are the landsat image, the pixel quality product, and the elevation model. (The difficulty is generating some of the flags, e.g. terrain shadow.)
|
|
54
|
+
|
|
55
|
+
Summary
|
|
56
|
+
-------
|
|
57
|
+
|
|
58
|
+
The summary product has multiple parts:
|
|
59
|
+
|
|
60
|
+
- a mean mosaic of the wofls (i.e. fraction of clear observations that are wet);
|
|
61
|
+
- a confidence estimate. This is a logistic function wrapping a linear combination (with published weights) of several inputs: 0. mean mosaic of the wofls, 1. multi-res valley bottom flatness, 2. MODIS open water likelihood, hydrological geofabric, 3. slope, 4-12. hydrological geofabric (boolean vectors), 13. Aus Stat Geog Standard (urban boolean).
|
|
62
|
+
- Filtered summary, i.e., mean mosaic clipped to always-dry where confidence is below a threshold. (Would also be interesting to see confidence applied as an opacity alpha channel to the mean mosaic?)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
Notes and ideas
|
|
66
|
+
===============
|
|
67
|
+
|
|
68
|
+
Profiling
|
|
69
|
+
---------
|
|
70
|
+
|
|
71
|
+
Implementation of potential optimisations was deliberately deferred, until memory, CPU and IO profiling could take place.
|
|
72
|
+
|
|
73
|
+
Results (below) indicate that memory is already within the 2GB/core available, that IO is not a significant bottleneck (before scaling), and that speed is unlikely to improve dramatically (since limited by intrinsically demanding aspects of the current terrain-shadow algorithm); therefore significant optimisation effort may not be warranted.
|
|
74
|
+
|
|
75
|
+
19/9/02016: Querying test cube and writing 4 tiles (16MB each). /usr/bin/time showed 1min30sec walltime, ~135% CPU usage, ~10% system (rather than user time), ~1.5GB max resident. cProfile time graph indicated 2.4% spent on imports, 4% on database queries, 26% on grid workflow (including >5% on rasterio read and 19% on rasterio reproject) and 65% on the core algorithm. The latter is dominated by the terrain filtering (56%, alongside 4.4% decision tree, 3.3% PQ, 1.1% EO filter). It includes 1.9% on the Sobel operation, 9.5% row shading (of which 8% is python code, as is 5.8% of shadows and slope), and 37% rotating the image (scipy). Dilation also totals 4%. Summary:
|
|
76
|
+
|
|
77
|
+
- 20% potential speedup by storing DSM in the same projection as EO, or by orchestrating execution to avoid reloading DSM redundantly.
|
|
78
|
+
- Most of the time is spent on terrain, but only 5-10% speedup plausible by better implementation.
|
|
79
|
+
- Most limiting factor is rotating the DSM (to approximately align with sunlight) but nontrivial to improve or mitigate this. (May or may not be amenable to cheaper interpolation methods or an algorithm that traverses the array differently.)
|
|
80
|
+
|
|
81
|
+
Overlaps
|
|
82
|
+
--------
|
|
83
|
+
The Landsats collect a swath of data as they pass over the continent.
|
|
84
|
+
Traditionally, each pass is segmented into overlapping scenes for processing
|
|
85
|
+
separately. This necessitates measures to avoid double-counting duplicated
|
|
86
|
+
observations. Potential alternate measures would include:
|
|
87
|
+
|
|
88
|
+
- Whole pass based processing. (Upstream software not yet available.)
|
|
89
|
+
- EO archive of reconstituted seamless passes,
|
|
90
|
+
e.g., fusing scene-overlaps during ingestion.
|
|
91
|
+
(Renounced in current datacube iteration.)
|
|
92
|
+
- Duplicate-free water extents, e.g., fusing the inputs to wofl generation.
|
|
93
|
+
(Interferes with potential use of scene middle pixel timestamp as a primary
|
|
94
|
+
key for matching EO scenes to wofls.)
|
|
95
|
+
- Downstream each user/application perform wofl fusing.
|
|
96
|
+
(Beyond capability of the generic API; requires sharing wofs-specific code.)
|
|
97
|
+
|
|
98
|
+
The usefulness of preserving observation-duplicates (e.g. for investigation of
|
|
99
|
+
sensitivity to uncontrolled upstream processing parameters) is narrow.
|
|
100
|
+
|
|
101
|
+
Classifier
|
|
102
|
+
----------
|
|
103
|
+
|
|
104
|
+
It may improve performance and readability to represent the decision tree as a numexpr statement (nested across multiple lines). This could additionally include some of the mask logic.
|
|
105
|
+
|
|
106
|
+
Ideally the PQ product might be a band in the EO product (and include terrain related bitflags).
|
|
107
|
+
|
|
108
|
+
Alternative algorithms are under development elsewhere.
|
|
109
|
+
|
|
110
|
+
Terrain
|
|
111
|
+
-------
|
|
112
|
+
|
|
113
|
+
Terrain algorithms usually begin with finding the gradient component along each of the two axes, typically by operating with a 3x3 kernel. One example is the Rook's case (simply using nearest neighbours on either side of the pixel, which turns out to be a 2nd order finite difference method). Another is the Sobel operator, which additionally applies smoothing along the orthogonal axis. Tang and Pilesjo 2011 showed these belong to a variety of methods which produce statistically similar results (different from a more naive and unbalanced method of differencing the central cell with one neighbour along each axis). Jones 1998 found the Rook's case to give the best accuracy (narrowly followed by Sobel), but the methodology (e.g. noise-free synthetic) may have been biased (to favour balanced methods with more compact footprints). Zhou and Liu 2004 added noise to a synthetic, confirming the Rook's case to be optimal in absence of noise but the Sobel operator was more robust to the noise.
|
|
114
|
+
|
|
115
|
+
Clouds
|
|
116
|
+
------
|
|
117
|
+
|
|
118
|
+
Currently, cloud and cloud shadow are detected per scene, which is suboptimal at contiguous boundaries.
|
|
119
|
+
|
|
120
|
+
Improved masking algorithms are anticipated, e.g. as median mosaics become available, or possibly incorporating weather data.
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
Packaging and Releases
|
|
124
|
+
======================
|
|
125
|
+
|
|
126
|
+
Versioning
|
|
127
|
+
----------
|
|
128
|
+
|
|
129
|
+
The version number is based on the **algorithm version
|
|
130
|
+
number**, which at the moment stands at 1.4. See the `CMI Record for the WOfS Algorithm
|
|
131
|
+
<http://cmi.ga.gov.au/node/166>`_.
|
|
132
|
+
|
|
133
|
+
For minor code changes not affecting the algorithm, increment the least significant digit of the version number.
|
|
134
|
+
|
|
135
|
+
Releases
|
|
136
|
+
--------
|
|
137
|
+
|
|
138
|
+
To release a new package of WOfS, `create a new release <https://github.com/GeoscienceAustralia/wofs/releases/new>`_
|
|
139
|
+
using GitHub, with a suitably tagged version number.
|
|
140
|
+
|
|
141
|
+
The Continuous Integration service will run tests, create source and binary distribution packages, and upload them
|
|
142
|
+
to https://packages.dea.ga.gov.au/.
|
|
143
|
+
|
|
144
|
+
To build a new package for WOfS, Then, from the base directory of the project run:
|
|
145
|
+
|
|
146
|
+
python setup.py sdist bdist_wheel
|
|
147
|
+
|
|
148
|
+
This will create a ``source distribution`` and a ``binary wheel`` distribution in the ``dist/`` directory.
|
|
149
|
+
|
|
150
|
+
To have the package included in the **DEA Environment Module** upload it to s3://datacube-core-deployment/wofs/ by
|
|
151
|
+
running:
|
|
152
|
+
|
|
153
|
+
aws s3 cp dist/ s3://datacube-core-deployment/wofs/ --recursive
|
|
154
|
+
|
wofs-1.6.6/check-code.sh
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
version: ${version}
|
|
2
|
+
|
|
3
|
+
location: '/g/data/fk4/datacube/002/WOfS/WOfS_25_2_1/netcdf'
|
|
4
|
+
file_path_template: '{tile_index[0]}_{tile_index[1]}/LS_WATER_3577_{tile_index[0]}_{tile_index[1]}_{start_time}_v{version}.nc'
|
|
5
|
+
|
|
6
|
+
product_definition:
|
|
7
|
+
name: wofs_albers
|
|
8
|
+
description: Historic Flood Mapping Water Observations from Space
|
|
9
|
+
managed: True
|
|
10
|
+
metadata_type: eo
|
|
11
|
+
metadata:
|
|
12
|
+
product_type: wofs
|
|
13
|
+
format:
|
|
14
|
+
name: NetCDF
|
|
15
|
+
storage:
|
|
16
|
+
crs: EPSG:3577
|
|
17
|
+
resolution:
|
|
18
|
+
x: 25
|
|
19
|
+
y: -25
|
|
20
|
+
tile_size:
|
|
21
|
+
x: 100000.0
|
|
22
|
+
y: 100000.0
|
|
23
|
+
driver: NetCDF CF
|
|
24
|
+
dimension_order: [time, y, x]
|
|
25
|
+
chunking:
|
|
26
|
+
x: 200
|
|
27
|
+
y: 200
|
|
28
|
+
time: 5
|
|
29
|
+
measurements:
|
|
30
|
+
- name: water
|
|
31
|
+
dtype: int16
|
|
32
|
+
nodata: 1
|
|
33
|
+
units: '1'
|
|
34
|
+
flags_definition:
|
|
35
|
+
dry:
|
|
36
|
+
bits: [7, 6, 5, 4, 3, 1, 0] # Ignore sea mask
|
|
37
|
+
description: Clear and dry
|
|
38
|
+
values: {0: true}
|
|
39
|
+
nodata:
|
|
40
|
+
bits: 0
|
|
41
|
+
description: No data
|
|
42
|
+
values: {0: false, 1: true}
|
|
43
|
+
noncontiguous:
|
|
44
|
+
bits: 1
|
|
45
|
+
description: At least one EO band is missing over over/undersaturated
|
|
46
|
+
values: {0: false, 1: true}
|
|
47
|
+
sea:
|
|
48
|
+
bits: 2
|
|
49
|
+
description: Sea
|
|
50
|
+
values: {0: false, 1: true}
|
|
51
|
+
terrain_or_low_angle:
|
|
52
|
+
bits: 3
|
|
53
|
+
description: Terrain shadow or low solar angle
|
|
54
|
+
values: {0: false, 1: true}
|
|
55
|
+
high_slope:
|
|
56
|
+
bits: 4
|
|
57
|
+
description: High slope
|
|
58
|
+
values: {0: false, 1: true}
|
|
59
|
+
cloud_shadow:
|
|
60
|
+
bits: 5
|
|
61
|
+
description: Cloud shadow
|
|
62
|
+
values: {0: false, 1: true}
|
|
63
|
+
cloud:
|
|
64
|
+
bits: 6
|
|
65
|
+
description: Cloudy
|
|
66
|
+
values: {0: false, 1: true}
|
|
67
|
+
water_observed:
|
|
68
|
+
bits: 7
|
|
69
|
+
description: Classified as water by the decision tree
|
|
70
|
+
values: {0: false, 1: true}
|
|
71
|
+
wet:
|
|
72
|
+
bits: [7, 6, 5, 4, 3, 1, 0] # Ignore sea mask
|
|
73
|
+
description: Clear and Wet
|
|
74
|
+
values: {128: true}
|
|
75
|
+
|
|
76
|
+
variable_params:
|
|
77
|
+
water:
|
|
78
|
+
zlib: True
|
|
79
|
+
fletcher32: True
|
|
80
|
+
chunksizes: [5, 200, 200]
|
|
81
|
+
attrs:
|
|
82
|
+
long_name: Water observation feature layer
|
|
83
|
+
coverage_content_type: "thematicClassification"
|
|
84
|
+
|
|
85
|
+
global_attributes:
|
|
86
|
+
cmi_id: "WO_25_2.1.5"
|
|
87
|
+
cmi_nid: "5"
|
|
88
|
+
title: "Water Observations from Space 25 v. 2.1.5"
|
|
89
|
+
summary: |
|
|
90
|
+
Water Observations from Space (WOfS) is a gridded dataset indicating areas where surface water has been observed using the Geoscience Australia (GA) Earth observation satellite data holdings. The current product (Version 2.1.5) includes observations taken between 1986 and 2017 (inclusive) from the Landsat 5, 7 and 8 satellites. WOfS covers all of mainland Australia and Tasmania but excludes off-shore Territories.
|
|
91
|
+
|
|
92
|
+
WOfS shows water observed for every Landsat-5, Landsat-7 and Landsat-8 image across Australia (excluding External Territories) for the period of 1987 to 2017. The dataset is updated quarterly and is expected to increase in update frequency in the future so that as a satellite acquires data, it will automatically be analysed for the presence of water and added to the WOfS product in near real time.
|
|
93
|
+
|
|
94
|
+
Data is provided as Water Observation Feature Layers (WOFLs), in a 1 to 1 relationship with the input satellite data. Hence there is one WOFL for each satellite dataset processed for the occurrence of water.
|
|
95
|
+
|
|
96
|
+
In the future, WOfS will be updated as new data are added. This is potentially possible because the dataset is produced using Digital Earth Australia, containing GA's entire Australian Landsat archive in a high performance computing environment at the National Computational Infrastructure at the Australian National University.
|
|
97
|
+
|
|
98
|
+
The Water Observations from Space product (WOfS) is a key component of the National Flood Risk Information Portal (NFRIP), developed by Geoscience Australia (GA). The objective of Water Observations from Space is to analyse GA's historic archive of satellite imagery to derive water observations, to help understand where flooding may have occurred in the past.
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
source: "Water Observations from Space Detection Algorithm v1.2"
|
|
102
|
+
institution: "Commonwealth of Australia (Geoscience Australia)"
|
|
103
|
+
keywords: "AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE,SURFACE WATER,FLOOD"
|
|
104
|
+
keywords_vocabulary: "GCMD"
|
|
105
|
+
product_version: "2.1.5"
|
|
106
|
+
publisher_email: earth.observation@ga.gov.au
|
|
107
|
+
publisher_name: Section Leader, Operations Section, NEMO, Geoscience Australia
|
|
108
|
+
publisher_url: http://www.ga.gov.au
|
|
109
|
+
license: "CC BY Attribution 4.0 International License"
|
|
110
|
+
cdm_data_type: "Grid"
|
|
111
|
+
product_suite: "Water Observations from Space 25m"
|
|
112
|
+
references: |
|
|
113
|
+
N. Mueller, A. Lewis, D. Roberts, S. Ring, R. Melrose, J. Sixsmith, L. Lymburner, A. McIntyre, P. Tan, S. Curnow, A. Ip, Water observations from space: Mapping surface water from 25 years of Landsat imagery across Australia, Remote Sensing of Environment, Volume 174, 1 March 2016, Pages 341-352, ISSN 0034-4257, http://dx.doi.org/10.1016/j.rse.2015.11.003. (http://www.sciencedirect.com/science/article/pii/S0034425715301929)
|