wofs 1.6.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. wofs-1.6.6/.github/workflows/push.yaml +66 -0
  2. wofs-1.6.6/.gitignore +100 -0
  3. wofs-1.6.6/.yamllint +22 -0
  4. wofs-1.6.6/PKG-INFO +184 -0
  5. wofs-1.6.6/README.rst +154 -0
  6. wofs-1.6.6/check-code.sh +9 -0
  7. wofs-1.6.6/config/wofs_albers.yaml +113 -0
  8. wofs-1.6.6/config/wofs_modified_albers.yaml +114 -0
  9. wofs-1.6.6/example.ipynb +131 -0
  10. wofs-1.6.6/pylintrc +241 -0
  11. wofs-1.6.6/pyproject.toml +6 -0
  12. wofs-1.6.6/runtests-docker.sh +7 -0
  13. wofs-1.6.6/setup.cfg +13 -0
  14. wofs-1.6.6/setup.py +50 -0
  15. wofs-1.6.6/tests/Test core algorithm.ipynb +481 -0
  16. wofs-1.6.6/tests/create_test_input_data.py +39 -0
  17. wofs-1.6.6/tests/sample_c3_sr.nc +0 -0
  18. wofs-1.6.6/tests/sample_wofl.nc +0 -0
  19. wofs-1.6.6/tests/test_bitmask_values.py +60 -0
  20. wofs-1.6.6/tests/test_terrain.py +39 -0
  21. wofs-1.6.6/tests/test_virtualproduct.py +62 -0
  22. wofs-1.6.6/tests/test_wofs.py +27 -0
  23. wofs-1.6.6/wofs/__init__.py +5 -0
  24. wofs-1.6.6/wofs/_version.py +16 -0
  25. wofs-1.6.6/wofs/boilerplate.py +11 -0
  26. wofs-1.6.6/wofs/classifier.py +232 -0
  27. wofs-1.6.6/wofs/constants.py +49 -0
  28. wofs-1.6.6/wofs/filters.py +167 -0
  29. wofs-1.6.6/wofs/terrain.py +150 -0
  30. wofs-1.6.6/wofs/virtualproduct.py +138 -0
  31. wofs-1.6.6/wofs/wofls.py +109 -0
  32. wofs-1.6.6/wofs/wofs_app.py +730 -0
  33. wofs-1.6.6/wofs-summary/confidence.ipynb +757 -0
  34. wofs-1.6.6/wofs-summary/confidence.py +341 -0
  35. wofs-1.6.6/wofs-summary/job.sh +20 -0
  36. wofs-1.6.6/wofs-summary/simple.py +203 -0
  37. wofs-1.6.6/wofs.egg-info/PKG-INFO +184 -0
  38. wofs-1.6.6/wofs.egg-info/SOURCES.txt +41 -0
  39. wofs-1.6.6/wofs.egg-info/dependency_links.txt +1 -0
  40. wofs-1.6.6/wofs.egg-info/entry_points.txt +2 -0
  41. wofs-1.6.6/wofs.egg-info/requires.txt +13 -0
  42. wofs-1.6.6/wofs.egg-info/top_level.txt +1 -0
@@ -0,0 +1,66 @@
1
+ name: Test and Release
2
+
3
+ on:
4
+ push:
5
+
6
+ jobs:
7
+ test:
8
+ runs-on: ubuntu-latest
9
+ strategy:
10
+ matrix:
11
+ python-version: [3.8, 3.9]
12
+ name: Python ${{ matrix.python-version }}
13
+
14
+ steps:
15
+ - uses: actions/checkout@v2
16
+ with:
17
+ fetch-depth: 0
18
+
19
+ - name: Install dependencies
20
+ run: |
21
+ sudo apt-get update && sudo apt-get install -f libudunits2-dev
22
+ pip install .[test]
23
+
24
+ - name: Run tests
25
+ run: ./check-code.sh
26
+
27
+ - name: Build package
28
+ run: |
29
+ python setup.py sdist bdist_wheel
30
+
31
+ - name: Upload artifacts
32
+ uses: actions/upload-artifact@v2
33
+ with:
34
+ name: packages
35
+ path: dist
36
+
37
+ - uses: codecov/codecov-action@v1
38
+ with:
39
+ env_vars: OS,PYTHON
40
+ file: ./coverage.xml
41
+
42
+ deploy-packages:
43
+ if: startsWith(github.ref, 'refs/tags/')
44
+ runs-on: ubuntu-latest
45
+ needs: test
46
+ steps:
47
+ - name: Download a single artifact
48
+ uses: actions/download-artifact@v2
49
+ with:
50
+ name: packages
51
+ path: dist
52
+
53
+ - name: Display directory structure of downloaded files
54
+ run: ls -lR
55
+
56
+ - name: Deploy packages
57
+ uses: jakejarvis/s3-sync-action@master
58
+ with:
59
+ args: --acl public-read --follow-symlinks
60
+ env:
61
+ AWS_S3_BUCKET: "datacube-core-deployment"
62
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
63
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
64
+ AWS_REGION: "ap-southeast-2" # optional: defaults to us-east-1
65
+ SOURCE_DIR: "dist" # optional: defaults to entire repository
66
+ DEST_DIR: "wofs"
wofs-1.6.6/.gitignore ADDED
@@ -0,0 +1,100 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+
5
+ # C extensions
6
+ *.so
7
+
8
+ # Distribution / packaging
9
+ .Python
10
+ env/
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ *.egg-info/
23
+ .installed.cfg
24
+ *.egg
25
+
26
+ # PyInstaller
27
+ # Usually these files are written by a python script from a template
28
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
29
+ *.manifest
30
+ *.spec
31
+
32
+ # Installer logs
33
+ pip-log.txt
34
+ pip-delete-this-directory.txt
35
+
36
+ # Unit test / coverage reports
37
+ htmlcov/
38
+ .tox/
39
+ .coverage
40
+ .coverage.*
41
+ .cache
42
+ nosetests.xml
43
+ coverage.xml
44
+ *,cover
45
+ .hypothesis
46
+
47
+ # Translations
48
+ *.mo
49
+ *.pot
50
+
51
+ # Django stuff:
52
+ *.log
53
+
54
+ # Sphinx documentation
55
+ docs/_build/
56
+
57
+ # PyBuilder
58
+ target/
59
+ .idea/
60
+
61
+ # iPython Notebook
62
+ .ipynb_checkpoints
63
+
64
+ # PBS Job manager output
65
+ *.[eo][0-9]*
66
+ log/*
67
+ !log/README.md
68
+
69
+ # images
70
+ *.tif
71
+ *.tiff
72
+
73
+
74
+ # pdf files
75
+ *.pdf
76
+
77
+ #LaTeX files
78
+ #
79
+ *.log
80
+ *.out
81
+ *.ps
82
+ *.aux
83
+
84
+ # archive files
85
+ *.tar
86
+ *.tar.gz
87
+ *.zip
88
+
89
+ # private, local files
90
+ local*
91
+ zlocal*
92
+ private*
93
+ scratch/*
94
+
95
+ .pytest_cache/
96
+ .vscode
97
+
98
+ wofs/_version.py
99
+ .benchmarks/
100
+ tests/.benchmarks/
wofs-1.6.6/.yamllint ADDED
@@ -0,0 +1,22 @@
1
+ extends: default
2
+
3
+ rules:
4
+ # We could enable these if we find a decent yaml autoformatter.
5
+ # (yamlfmt in pip is too buggy at the moment for our files)
6
+ #
7
+ # The effort of fixing them by hand, especially auto-generated yamls, is not
8
+ # currently worth it.
9
+
10
+ # Many tools (eg. generated secure keys) don't output wrapped lines.
11
+ line-length: disable
12
+
13
+ # The existing documents will need to be cleaned up first
14
+ indentation: disable
15
+
16
+ # Pedantry & existing docs don't have it.
17
+ document-start: disable
18
+
19
+ # Warning that truthy values are not quoted.
20
+ # All documents currently use "True" without quotes, so this would be a
21
+ # larger change across almost every doc.
22
+ truthy: disable
wofs-1.6.6/PKG-INFO ADDED
@@ -0,0 +1,184 @@
1
+ Metadata-Version: 2.1
2
+ Name: wofs
3
+ Version: 1.6.6
4
+ Summary: Water Observations from Space - Digital Earth Australia
5
+ Home-page: https://github.com/GeoscienceAustralia/wofs
6
+ Author: Geoscience Australia
7
+ Maintainer: Geoscience Australia
8
+ Maintainer-email:
9
+ License: Apache License 2.0
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: Apache Software License
13
+ Classifier: Programming Language :: Python :: 3.6
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Topic :: Scientific/Engineering :: GIS
17
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
18
+ Requires-Dist: datacube
19
+ Requires-Dist: scipy
20
+ Requires-Dist: ephem
21
+ Requires-Dist: xarray>=0.14.1
22
+ Provides-Extra: test
23
+ Requires-Dist: pytest; extra == "test"
24
+ Requires-Dist: pytest-cov; extra == "test"
25
+ Requires-Dist: mock; extra == "test"
26
+ Requires-Dist: pylint; extra == "test"
27
+ Requires-Dist: hypothesis; extra == "test"
28
+ Requires-Dist: compliance-checker; extra == "test"
29
+ Requires-Dist: black; extra == "test"
30
+
31
+ Water Observation from Space (WOfS)
32
+ ====================================
33
+
34
+ :Disclaimer:
35
+ This repository is *in development*, use at your own risk.
36
+
37
+ :License:
38
+ The Apache 2.0 license applies to this open source code.
39
+
40
+
41
+ This WOfS version applies the original published WOfS decision-tree algorithm,
42
+ but is updated to use the Open Data Cube and
43
+ `xarray <http://xarray.pydata.org/en/stable/>`_ for data access.
44
+
45
+ Specifically, this version also decouples the production of water extent tiles
46
+ from the production of the statistical summary mosaics, and is intended to
47
+ improve consistency with other datacube applications (e.g., parallelisation
48
+ of the workflow employs *distributed* rather than *luigi*).
49
+
50
+ Codebase outline
51
+ ----------------
52
+
53
+ The water-specific code (as distinct from packaging boilerplate) is located
54
+ in the "wofs" directory, other than metadata that may be located in the the
55
+ config yaml.
56
+
57
+ Installation
58
+ ============
59
+
60
+ The WOfS package can be installed by running:
61
+
62
+ pip install --index-url https://packages.dea.ga.gov.au/ wofs
63
+
64
+
65
+ For Digital Earth Australia Users
66
+ ---------------------------------
67
+
68
+ WOfS is available as a part of Digital Earth Australia environment modules on the NCI. These can be used
69
+ after logging into the NCI by running:
70
+
71
+ module load dea
72
+
73
+ Algorithm
74
+ =========
75
+
76
+ WOFLs
77
+ -----
78
+
79
+ Water Observation Feature Layers are the temporal foliation of water extents.
80
+ These consist of an 8-bit integer raster band.
81
+
82
+ - **Decision tree:** The standard classifier is *band maths* performed on 6 EO source bands (TM 1-5, 7). A published tree with 21 nodes, producing boolean output, comprising of thresholds applied to three raw bands (TM 1, 3, 7) and three band-pair ratio-indices (NDI 52, 43, 72).
83
+ - **Filter masks:** various flags are accumulated onto the output band. Inputs are the landsat image, the pixel quality product, and the elevation model. (The difficulty is generating some of the flags, e.g. terrain shadow.)
84
+
85
+ Summary
86
+ -------
87
+
88
+ The summary product has multiple parts:
89
+
90
+ - a mean mosaic of the wofls (i.e. fraction of clear observations that are wet);
91
+ - a confidence estimate. This is a logistic function wrapping a linear combination (with published weights) of several inputs: 0. mean mosaic of the wofls, 1. multi-res valley bottom flatness, 2. MODIS open water likelihood, hydrological geofabric, 3. slope, 4-12. hydrological geofabric (boolean vectors), 13. Aus Stat Geog Standard (urban boolean).
92
+ - Filtered summary, i.e., mean mosaic clipped to always-dry where confidence is below a threshold. (Would also be interesting to see confidence applied as an opacity alpha channel to the mean mosaic?)
93
+
94
+
95
+ Notes and ideas
96
+ ===============
97
+
98
+ Profiling
99
+ ---------
100
+
101
+ Implementation of potential optimisations was deliberately deferred, until memory, CPU and IO profiling could take place.
102
+
103
+ Results (below) indicate that memory is already within the 2GB/core available, that IO is not a significant bottleneck (before scaling), and that speed is unlikely to improve dramatically (since limited by intrinsically demanding aspects of the current terrain-shadow algorithm); therefore significant optimisation effort may not be warranted.
104
+
105
+ 19/9/02016: Querying test cube and writing 4 tiles (16MB each). /usr/bin/time showed 1min30sec walltime, ~135% CPU usage, ~10% system (rather than user time), ~1.5GB max resident. cProfile time graph indicated 2.4% spent on imports, 4% on database queries, 26% on grid workflow (including >5% on rasterio read and 19% on rasterio reproject) and 65% on the core algorithm. The latter is dominated by the terrain filtering (56%, alongside 4.4% decision tree, 3.3% PQ, 1.1% EO filter). It includes 1.9% on the Sobel operation, 9.5% row shading (of which 8% is python code, as is 5.8% of shadows and slope), and 37% rotating the image (scipy). Dilation also totals 4%. Summary:
106
+
107
+ - 20% potential speedup by storing DSM in the same projection as EO, or by orchestrating execution to avoid reloading DSM redundantly.
108
+ - Most of the time is spent on terrain, but only 5-10% speedup plausible by better implementation.
109
+ - Most limiting factor is rotating the DSM (to approximately align with sunlight) but nontrivial to improve or mitigate this. (May or may not be amenable to cheaper interpolation methods or an algorithm that traverses the array differently.)
110
+
111
+ Overlaps
112
+ --------
113
+ The Landsats collect a swath of data as they pass over the continent.
114
+ Traditionally, each pass is segmented into overlapping scenes for processing
115
+ separately. This necessitates measures to avoid double-counting duplicated
116
+ observations. Potential alternate measures would include:
117
+
118
+ - Whole pass based processing. (Upstream software not yet available.)
119
+ - EO archive of reconstituted seamless passes,
120
+ e.g., fusing scene-overlaps during ingestion.
121
+ (Renounced in current datacube iteration.)
122
+ - Duplicate-free water extents, e.g., fusing the inputs to wofl generation.
123
+ (Interferes with potential use of scene middle pixel timestamp as a primary
124
+ key for matching EO scenes to wofls.)
125
+ - Downstream each user/application perform wofl fusing.
126
+ (Beyond capability of the generic API; requires sharing wofs-specific code.)
127
+
128
+ The usefulness of preserving observation-duplicates (e.g. for investigation of
129
+ sensitivity to uncontrolled upstream processing parameters) is narrow.
130
+
131
+ Classifier
132
+ ----------
133
+
134
+ It may improve performance and readability to represent the decision tree as a numexpr statement (nested across multiple lines). This could additionally include some of the mask logic.
135
+
136
+ Ideally the PQ product might be a band in the EO product (and include terrain related bitflags).
137
+
138
+ Alternative algorithms are under development elsewhere.
139
+
140
+ Terrain
141
+ -------
142
+
143
+ Terrain algorithms usually begin with finding the gradient component along each of the two axes, typically by operating with a 3x3 kernel. One example is the Rook's case (simply using nearest neighbours on either side of the pixel, which turns out to be a 2nd order finite difference method). Another is the Sobel operator, which additionally applies smoothing along the orthogonal axis. Tang and Pilesjo 2011 showed these belong to a variety of methods which produce statistically similar results (different from a more naive and unbalanced method of differencing the central cell with one neighbour along each axis). Jones 1998 found the Rook's case to give the best accuracy (narrowly followed by Sobel), but the methodology (e.g. noise-free synthetic) may have been biased (to favour balanced methods with more compact footprints). Zhou and Liu 2004 added noise to a synthetic, confirming the Rook's case to be optimal in absence of noise but the Sobel operator was more robust to the noise.
144
+
145
+ Clouds
146
+ ------
147
+
148
+ Currently, cloud and cloud shadow are detected per scene, which is suboptimal at contiguous boundaries.
149
+
150
+ Improved masking algorithms are anticipated, e.g. as median mosaics become available, or possibly incorporating weather data.
151
+
152
+
153
+ Packaging and Releases
154
+ ======================
155
+
156
+ Versioning
157
+ ----------
158
+
159
+ The version number is based on the **algorithm version
160
+ number**, which at the moment stands at 1.4. See the `CMI Record for the WOfS Algorithm
161
+ <http://cmi.ga.gov.au/node/166>`_.
162
+
163
+ For minor code changes not affecting the algorithm, increment the least significant digit of the version number.
164
+
165
+ Releases
166
+ --------
167
+
168
+ To release a new package of WOfS, `create a new release <https://github.com/GeoscienceAustralia/wofs/releases/new>`_
169
+ using GitHub, with a suitably tagged version number.
170
+
171
+ The Continuous Integration service will run tests, create source and binary distribution packages, and upload them
172
+ to https://packages.dea.ga.gov.au/.
173
+
174
+ To build a new package for WOfS, Then, from the base directory of the project run:
175
+
176
+ python setup.py sdist bdist_wheel
177
+
178
+ This will create a ``source distribution`` and a ``binary wheel`` distribution in the ``dist/`` directory.
179
+
180
+ To have the package included in the **DEA Environment Module** upload it to s3://datacube-core-deployment/wofs/ by
181
+ running:
182
+
183
+ aws s3 cp dist/ s3://datacube-core-deployment/wofs/ --recursive
184
+
wofs-1.6.6/README.rst ADDED
@@ -0,0 +1,154 @@
1
+ Water Observation from Space (WOfS)
2
+ ====================================
3
+
4
+ :Disclaimer:
5
+ This repository is *in development*, use at your own risk.
6
+
7
+ :License:
8
+ The Apache 2.0 license applies to this open source code.
9
+
10
+
11
+ This WOfS version applies the original published WOfS decision-tree algorithm,
12
+ but is updated to use the Open Data Cube and
13
+ `xarray <http://xarray.pydata.org/en/stable/>`_ for data access.
14
+
15
+ Specifically, this version also decouples the production of water extent tiles
16
+ from the production of the statistical summary mosaics, and is intended to
17
+ improve consistency with other datacube applications (e.g., parallelisation
18
+ of the workflow employs *distributed* rather than *luigi*).
19
+
20
+ Codebase outline
21
+ ----------------
22
+
23
+ The water-specific code (as distinct from packaging boilerplate) is located
24
+ in the "wofs" directory, other than metadata that may be located in the the
25
+ config yaml.
26
+
27
+ Installation
28
+ ============
29
+
30
+ The WOfS package can be installed by running:
31
+
32
+ pip install --index-url https://packages.dea.ga.gov.au/ wofs
33
+
34
+
35
+ For Digital Earth Australia Users
36
+ ---------------------------------
37
+
38
+ WOfS is available as a part of Digital Earth Australia environment modules on the NCI. These can be used
39
+ after logging into the NCI by running:
40
+
41
+ module load dea
42
+
43
+ Algorithm
44
+ =========
45
+
46
+ WOFLs
47
+ -----
48
+
49
+ Water Observation Feature Layers are the temporal foliation of water extents.
50
+ These consist of an 8-bit integer raster band.
51
+
52
+ - **Decision tree:** The standard classifier is *band maths* performed on 6 EO source bands (TM 1-5, 7). A published tree with 21 nodes, producing boolean output, comprising of thresholds applied to three raw bands (TM 1, 3, 7) and three band-pair ratio-indices (NDI 52, 43, 72).
53
+ - **Filter masks:** various flags are accumulated onto the output band. Inputs are the landsat image, the pixel quality product, and the elevation model. (The difficulty is generating some of the flags, e.g. terrain shadow.)
54
+
55
+ Summary
56
+ -------
57
+
58
+ The summary product has multiple parts:
59
+
60
+ - a mean mosaic of the wofls (i.e. fraction of clear observations that are wet);
61
+ - a confidence estimate. This is a logistic function wrapping a linear combination (with published weights) of several inputs: 0. mean mosaic of the wofls, 1. multi-res valley bottom flatness, 2. MODIS open water likelihood, hydrological geofabric, 3. slope, 4-12. hydrological geofabric (boolean vectors), 13. Aus Stat Geog Standard (urban boolean).
62
+ - Filtered summary, i.e., mean mosaic clipped to always-dry where confidence is below a threshold. (Would also be interesting to see confidence applied as an opacity alpha channel to the mean mosaic?)
63
+
64
+
65
+ Notes and ideas
66
+ ===============
67
+
68
+ Profiling
69
+ ---------
70
+
71
+ Implementation of potential optimisations was deliberately deferred, until memory, CPU and IO profiling could take place.
72
+
73
+ Results (below) indicate that memory is already within the 2GB/core available, that IO is not a significant bottleneck (before scaling), and that speed is unlikely to improve dramatically (since limited by intrinsically demanding aspects of the current terrain-shadow algorithm); therefore significant optimisation effort may not be warranted.
74
+
75
+ 19/9/02016: Querying test cube and writing 4 tiles (16MB each). /usr/bin/time showed 1min30sec walltime, ~135% CPU usage, ~10% system (rather than user time), ~1.5GB max resident. cProfile time graph indicated 2.4% spent on imports, 4% on database queries, 26% on grid workflow (including >5% on rasterio read and 19% on rasterio reproject) and 65% on the core algorithm. The latter is dominated by the terrain filtering (56%, alongside 4.4% decision tree, 3.3% PQ, 1.1% EO filter). It includes 1.9% on the Sobel operation, 9.5% row shading (of which 8% is python code, as is 5.8% of shadows and slope), and 37% rotating the image (scipy). Dilation also totals 4%. Summary:
76
+
77
+ - 20% potential speedup by storing DSM in the same projection as EO, or by orchestrating execution to avoid reloading DSM redundantly.
78
+ - Most of the time is spent on terrain, but only 5-10% speedup plausible by better implementation.
79
+ - Most limiting factor is rotating the DSM (to approximately align with sunlight) but nontrivial to improve or mitigate this. (May or may not be amenable to cheaper interpolation methods or an algorithm that traverses the array differently.)
80
+
81
+ Overlaps
82
+ --------
83
+ The Landsats collect a swath of data as they pass over the continent.
84
+ Traditionally, each pass is segmented into overlapping scenes for processing
85
+ separately. This necessitates measures to avoid double-counting duplicated
86
+ observations. Potential alternate measures would include:
87
+
88
+ - Whole pass based processing. (Upstream software not yet available.)
89
+ - EO archive of reconstituted seamless passes,
90
+ e.g., fusing scene-overlaps during ingestion.
91
+ (Renounced in current datacube iteration.)
92
+ - Duplicate-free water extents, e.g., fusing the inputs to wofl generation.
93
+ (Interferes with potential use of scene middle pixel timestamp as a primary
94
+ key for matching EO scenes to wofls.)
95
+ - Downstream each user/application perform wofl fusing.
96
+ (Beyond capability of the generic API; requires sharing wofs-specific code.)
97
+
98
+ The usefulness of preserving observation-duplicates (e.g. for investigation of
99
+ sensitivity to uncontrolled upstream processing parameters) is narrow.
100
+
101
+ Classifier
102
+ ----------
103
+
104
+ It may improve performance and readability to represent the decision tree as a numexpr statement (nested across multiple lines). This could additionally include some of the mask logic.
105
+
106
+ Ideally the PQ product might be a band in the EO product (and include terrain related bitflags).
107
+
108
+ Alternative algorithms are under development elsewhere.
109
+
110
+ Terrain
111
+ -------
112
+
113
+ Terrain algorithms usually begin with finding the gradient component along each of the two axes, typically by operating with a 3x3 kernel. One example is the Rook's case (simply using nearest neighbours on either side of the pixel, which turns out to be a 2nd order finite difference method). Another is the Sobel operator, which additionally applies smoothing along the orthogonal axis. Tang and Pilesjo 2011 showed these belong to a variety of methods which produce statistically similar results (different from a more naive and unbalanced method of differencing the central cell with one neighbour along each axis). Jones 1998 found the Rook's case to give the best accuracy (narrowly followed by Sobel), but the methodology (e.g. noise-free synthetic) may have been biased (to favour balanced methods with more compact footprints). Zhou and Liu 2004 added noise to a synthetic, confirming the Rook's case to be optimal in absence of noise but the Sobel operator was more robust to the noise.
114
+
115
+ Clouds
116
+ ------
117
+
118
+ Currently, cloud and cloud shadow are detected per scene, which is suboptimal at contiguous boundaries.
119
+
120
+ Improved masking algorithms are anticipated, e.g. as median mosaics become available, or possibly incorporating weather data.
121
+
122
+
123
+ Packaging and Releases
124
+ ======================
125
+
126
+ Versioning
127
+ ----------
128
+
129
+ The version number is based on the **algorithm version
130
+ number**, which at the moment stands at 1.4. See the `CMI Record for the WOfS Algorithm
131
+ <http://cmi.ga.gov.au/node/166>`_.
132
+
133
+ For minor code changes not affecting the algorithm, increment the least significant digit of the version number.
134
+
135
+ Releases
136
+ --------
137
+
138
+ To release a new package of WOfS, `create a new release <https://github.com/GeoscienceAustralia/wofs/releases/new>`_
139
+ using GitHub, with a suitably tagged version number.
140
+
141
+ The Continuous Integration service will run tests, create source and binary distribution packages, and upload them
142
+ to https://packages.dea.ga.gov.au/.
143
+
144
+ To build a new package for WOfS, Then, from the base directory of the project run:
145
+
146
+ python setup.py sdist bdist_wheel
147
+
148
+ This will create a ``source distribution`` and a ``binary wheel`` distribution in the ``dist/`` directory.
149
+
150
+ To have the package included in the **DEA Environment Module** upload it to s3://datacube-core-deployment/wofs/ by
151
+ running:
152
+
153
+ aws s3 cp dist/ s3://datacube-core-deployment/wofs/ --recursive
154
+
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -eu
4
+ set -x
5
+
6
+ # Fix me in later PR
7
+ black --check . || true
8
+
9
+ pytest tests
@@ -0,0 +1,113 @@
1
+ version: ${version}
2
+
3
+ location: '/g/data/fk4/datacube/002/WOfS/WOfS_25_2_1/netcdf'
4
+ file_path_template: '{tile_index[0]}_{tile_index[1]}/LS_WATER_3577_{tile_index[0]}_{tile_index[1]}_{start_time}_v{version}.nc'
5
+
6
+ product_definition:
7
+ name: wofs_albers
8
+ description: Historic Flood Mapping Water Observations from Space
9
+ managed: True
10
+ metadata_type: eo
11
+ metadata:
12
+ product_type: wofs
13
+ format:
14
+ name: NetCDF
15
+ storage:
16
+ crs: EPSG:3577
17
+ resolution:
18
+ x: 25
19
+ y: -25
20
+ tile_size:
21
+ x: 100000.0
22
+ y: 100000.0
23
+ driver: NetCDF CF
24
+ dimension_order: [time, y, x]
25
+ chunking:
26
+ x: 200
27
+ y: 200
28
+ time: 5
29
+ measurements:
30
+ - name: water
31
+ dtype: int16
32
+ nodata: 1
33
+ units: '1'
34
+ flags_definition:
35
+ dry:
36
+ bits: [7, 6, 5, 4, 3, 1, 0] # Ignore sea mask
37
+ description: Clear and dry
38
+ values: {0: true}
39
+ nodata:
40
+ bits: 0
41
+ description: No data
42
+ values: {0: false, 1: true}
43
+ noncontiguous:
44
+ bits: 1
45
+ description: At least one EO band is missing over over/undersaturated
46
+ values: {0: false, 1: true}
47
+ sea:
48
+ bits: 2
49
+ description: Sea
50
+ values: {0: false, 1: true}
51
+ terrain_or_low_angle:
52
+ bits: 3
53
+ description: Terrain shadow or low solar angle
54
+ values: {0: false, 1: true}
55
+ high_slope:
56
+ bits: 4
57
+ description: High slope
58
+ values: {0: false, 1: true}
59
+ cloud_shadow:
60
+ bits: 5
61
+ description: Cloud shadow
62
+ values: {0: false, 1: true}
63
+ cloud:
64
+ bits: 6
65
+ description: Cloudy
66
+ values: {0: false, 1: true}
67
+ water_observed:
68
+ bits: 7
69
+ description: Classified as water by the decision tree
70
+ values: {0: false, 1: true}
71
+ wet:
72
+ bits: [7, 6, 5, 4, 3, 1, 0] # Ignore sea mask
73
+ description: Clear and Wet
74
+ values: {128: true}
75
+
76
+ variable_params:
77
+ water:
78
+ zlib: True
79
+ fletcher32: True
80
+ chunksizes: [5, 200, 200]
81
+ attrs:
82
+ long_name: Water observation feature layer
83
+ coverage_content_type: "thematicClassification"
84
+
85
+ global_attributes:
86
+ cmi_id: "WO_25_2.1.5"
87
+ cmi_nid: "5"
88
+ title: "Water Observations from Space 25 v. 2.1.5"
89
+ summary: |
90
+ Water Observations from Space (WOfS) is a gridded dataset indicating areas where surface water has been observed using the Geoscience Australia (GA) Earth observation satellite data holdings. The current product (Version 2.1.5) includes observations taken between 1986 and 2017 (inclusive) from the Landsat 5, 7 and 8 satellites. WOfS covers all of mainland Australia and Tasmania but excludes off-shore Territories.
91
+
92
+ WOfS shows water observed for every Landsat-5, Landsat-7 and Landsat-8 image across Australia (excluding External Territories) for the period of 1987 to 2017. The dataset is updated quarterly and is expected to increase in update frequency in the future so that as a satellite acquires data, it will automatically be analysed for the presence of water and added to the WOfS product in near real time.
93
+
94
+ Data is provided as Water Observation Feature Layers (WOFLs), in a 1 to 1 relationship with the input satellite data. Hence there is one WOFL for each satellite dataset processed for the occurrence of water.
95
+
96
+ In the future, WOfS will be updated as new data are added. This is potentially possible because the dataset is produced using Digital Earth Australia, containing GA's entire Australian Landsat archive in a high performance computing environment at the National Computational Infrastructure at the Australian National University.
97
+
98
+ The Water Observations from Space product (WOfS) is a key component of the National Flood Risk Information Portal (NFRIP), developed by Geoscience Australia (GA). The objective of Water Observations from Space is to analyse GA's historic archive of satellite imagery to derive water observations, to help understand where flooding may have occurred in the past.
99
+
100
+
101
+ source: "Water Observations from Space Detection Algorithm v1.2"
102
+ institution: "Commonwealth of Australia (Geoscience Australia)"
103
+ keywords: "AU/GA,NASA/GSFC/SED/ESD/LANDSAT,ETM+,TM,OLI,EARTH SCIENCE,SURFACE WATER,FLOOD"
104
+ keywords_vocabulary: "GCMD"
105
+ product_version: "2.1.5"
106
+ publisher_email: earth.observation@ga.gov.au
107
+ publisher_name: Section Leader, Operations Section, NEMO, Geoscience Australia
108
+ publisher_url: http://www.ga.gov.au
109
+ license: "CC BY Attribution 4.0 International License"
110
+ cdm_data_type: "Grid"
111
+ product_suite: "Water Observations from Space 25m"
112
+ references: |
113
+ N. Mueller, A. Lewis, D. Roberts, S. Ring, R. Melrose, J. Sixsmith, L. Lymburner, A. McIntyre, P. Tan, S. Curnow, A. Ip, Water observations from space: Mapping surface water from 25 years of Landsat imagery across Australia, Remote Sensing of Environment, Volume 174, 1 March 2016, Pages 341-352, ISSN 0034-4257, http://dx.doi.org/10.1016/j.rse.2015.11.003. (http://www.sciencedirect.com/science/article/pii/S0034425715301929)