grstbx 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- grstbx/__init__.py +44 -0
- grstbx/app.py +580 -0
- grstbx/data/OWT_centroids.nc +0 -0
- grstbx/data/OWT_mean_spec.csv +2511 -0
- grstbx/data/OWT_mean_spec_v01.csv +2511 -0
- grstbx/data/Thuillier_2003_0.3nm.dat +8228 -0
- grstbx/data/__init__.py +0 -0
- grstbx/datalake.py +109 -0
- grstbx/driver.py +532 -0
- grstbx/masking.py +222 -0
- grstbx/matchup.py +129 -0
- grstbx/subset.py +261 -0
- grstbx/utils.py +556 -0
- grstbx/visual.py +1374 -0
- grstbx-2.4.0.dist-info/METADATA +136 -0
- grstbx-2.4.0.dist-info/RECORD +20 -0
- grstbx-2.4.0.dist-info/WHEEL +5 -0
- grstbx-2.4.0.dist-info/entry_points.txt +2 -0
- grstbx-2.4.0.dist-info/licenses/LICENSE +674 -0
- grstbx-2.4.0.dist-info/top_level.txt +1 -0
grstbx/__init__.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
'''
|
|
2
|
+
version 1.0.3: fix for reprojection in any system,
|
|
3
|
+
specially useful for reprojection in pseudo-mercator EPSG:3857
|
|
4
|
+
for visualization with web mapping tools (e.g., OpenStreetViews, google map)
|
|
5
|
+
|
|
6
|
+
version 1.0.4: add tools for L2B handling
|
|
7
|
+
version 1.0.5: revisit datacube and raster object for multi-tile access
|
|
8
|
+
version 2.0.0: transition to GRS V2
|
|
9
|
+
v2.0.1: fix for gdal projection for accepted dtype, fix for dashboard visu
|
|
10
|
+
v2.0.2: small changes for the visu devices
|
|
11
|
+
v2.0.3: (2026-04-08) add tool option for masking / bitmask flagging
|
|
12
|
+
v2.0.4: code optimization and documentation; lazy loading of the visual module
|
|
13
|
+
v2.1.0: code optimization and update with pyproject.toml
|
|
14
|
+
v2.1.1: open Zarr v3 multiscale (pyramid) images, groups '0', '1', '2', ...
|
|
15
|
+
v2.2.0: crop L2A products to an AOI and export the subsets (grstbx.subset); ancillary group of Zarr products;
|
|
16
|
+
map helpers utm_projection and plot_rgb
|
|
17
|
+
packaging: dependency lower bounds, extras (notebook, regrid, docs, all), environment.yml
|
|
18
|
+
v2.3.0: multiscale viewers (grstbx.visual): ViewSpectral / ViewParam read only the visible pixels at screen
|
|
19
|
+
resolution from Zarr pyramids (or levels computed on demand), RGB composite, spectra of the drawn
|
|
20
|
+
points, free basemaps (CARTO, OSM, OpenTopoMap, Esri, Stamen via Stadia Maps)
|
|
21
|
+
v2.3.1: add live stream to visualize L2A data and reflectance spectra interactively
|
|
22
|
+
v2.3.2: matchups between satellite and in situ data (grstbx.matchup): get_matchups, matchup_metrics, plot_matchup
|
|
23
|
+
|
|
24
|
+
'''
|
|
25
|
+
|
|
26
|
+
__version__ = '2.4.0'
|
|
27
|
+
|
|
28
|
+
from .driver import L2grs, open_zarr_image, open_zarr_ancillary, zarr_levels, open_hgrs_l2a, is_hgrs
|
|
29
|
+
#from .driver_v1 import l2grs_v1
|
|
30
|
+
|
|
31
|
+
from .masking import Masking
|
|
32
|
+
from .utils import *
|
|
33
|
+
from .datalake import SelectFiles
|
|
34
|
+
from .subset import open_l2a, aoi_from_box, crop_l2a, export_l2a, export_rrs_geotiff, subset_path, crop_and_export
|
|
35
|
+
from .matchup import matchup_metrics, plot_matchup, get_matchups
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def __getattr__(name):
|
|
39
|
+
# import the (slow to load) holoviews/datashader-based viewers only when used
|
|
40
|
+
if name == 'visual':
|
|
41
|
+
import importlib
|
|
42
|
+
return importlib.import_module('.visual', __name__)
|
|
43
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
44
|
+
|
grstbx/app.py
ADDED
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Web dashboard to explore GRS (or hGRS) products: L2A images with :class:`~grstbx.visual.ViewSpectral`,
|
|
3
|
+
L2B images with :class:`~grstbx.visual.ViewParam`, one product, a time series of products or a
|
|
4
|
+
multi-temporal datacube.
|
|
5
|
+
|
|
6
|
+
The dashboard is a `Panel <https://panel.holoviz.org>`_ application served by a Python process, which
|
|
7
|
+
reads the pixels displayed on the screen and the spectra or time series under the mouse: start it with
|
|
8
|
+
|
|
9
|
+
.. code-block:: console
|
|
10
|
+
|
|
11
|
+
grstbx-viewer # empty dashboard, choose the products in the sidebar
|
|
12
|
+
grstbx-viewer /data/S2A_MSIL2AGRS_..._V3.0.1.zarr --root /data/satellite
|
|
13
|
+
grstbx-viewer "/data/L2B/BEFR/2024/*/*/*.nc" # time series of L2B products (glob pattern)
|
|
14
|
+
grstbx-viewer --address 0.0.0.0 --port 5006 --allow-websocket-origin myserver:5006
|
|
15
|
+
|
|
16
|
+
or from Python with :func:`serve`. The products are read on the machine running the server: with
|
|
17
|
+
``--address 0.0.0.0``, the dashboard is reachable from other computers of the network.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import glob
|
|
22
|
+
import io
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import warnings
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
import pandas as pd
|
|
29
|
+
import panel as pn
|
|
30
|
+
import xarray as xr
|
|
31
|
+
|
|
32
|
+
from .driver import is_hgrs, is_zarr
|
|
33
|
+
|
|
34
|
+
ACCEPTED = ('GRS L2A or L2B Zarr stores (`*.zarr`), GRS L2A NetCDF product folders (`<name>/` with '
|
|
35
|
+
'`<name>.nc` and `<name>_anc.nc`), GRS L2B NetCDF files, hGRS L2A NetCDF files (EnMAP, PRISMA), '
|
|
36
|
+
'or a NetCDF datacube with a `time` dimension')
|
|
37
|
+
|
|
38
|
+
# variables of the L2B products that are not parameters to display
|
|
39
|
+
NOT_PARAMETERS = ('flags', 'mask', 'spatial_ref', 'dem', 'valid_pix_prop')
|
|
40
|
+
# order of the L2B parameters in the viewer (by prefix), the other ones come last
|
|
41
|
+
PARAMETER_ORDER = ('Chla', 'SPM', 'TURB', 'acdom', 'Kd')
|
|
42
|
+
# datacubes smaller than this (bytes) are loaded in memory: fast time series under the mouse and exports
|
|
43
|
+
MAX_IN_MEMORY = 1e9
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# -- opening of the products ------------------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
def is_product(path):
|
|
49
|
+
"""True if ``path`` can be opened by :func:`open_products`: Zarr store, NetCDF file, or GRS NetCDF
|
|
50
|
+
product folder ``<name>/`` holding ``<name>.nc``."""
|
|
51
|
+
path = str(path).rstrip('/')
|
|
52
|
+
name = os.path.basename(path)
|
|
53
|
+
return is_zarr(path) or (os.path.isfile(path) and path.endswith('.nc')) or (
|
|
54
|
+
os.path.isdir(path) and os.path.isfile(os.path.join(path, name + '.nc')))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def expand_paths(paths):
|
|
58
|
+
"""
|
|
59
|
+
List of products from paths and glob patterns.
|
|
60
|
+
|
|
61
|
+
:param paths: path, glob pattern (e.g. ``'/data/L2B/BEFR/2024/*/*/*.nc'``), or list of them, or text
|
|
62
|
+
with one per line
|
|
63
|
+
:return: list of existing paths (patterns expanded and sorted), without duplicates
|
|
64
|
+
"""
|
|
65
|
+
if isinstance(paths, (str, os.PathLike)):
|
|
66
|
+
paths = str(paths).splitlines()
|
|
67
|
+
out = []
|
|
68
|
+
for item in paths:
|
|
69
|
+
item = str(item).strip().rstrip('/')
|
|
70
|
+
if not item:
|
|
71
|
+
continue
|
|
72
|
+
matches = sorted(glob.glob(item)) if glob.has_magic(item) else [item]
|
|
73
|
+
if not matches:
|
|
74
|
+
raise FileNotFoundError(f'no product matches {item}')
|
|
75
|
+
for match in matches:
|
|
76
|
+
if not os.path.exists(match):
|
|
77
|
+
raise FileNotFoundError(match)
|
|
78
|
+
if match not in out:
|
|
79
|
+
out.append(match)
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _image_time(ds, path):
|
|
84
|
+
"""Acquisition time of an image: ``time`` coordinate, attributes or date in the product name."""
|
|
85
|
+
if 'time' in ds.coords and ds['time'].size == 1:
|
|
86
|
+
return np.datetime64(pd.Timestamp(ds['time'].values.ravel()[0]).tz_localize(None), 'ns')
|
|
87
|
+
for attr in ('acquisition_date', 'start_date', 'PRODUCT_START_TIME'):
|
|
88
|
+
if attr in ds.attrs:
|
|
89
|
+
return np.datetime64(pd.Timestamp(ds.attrs[attr]).tz_localize(None), 'ns')
|
|
90
|
+
match = re.search(r'(\d{8}T\d{6})', os.path.basename(path))
|
|
91
|
+
if match:
|
|
92
|
+
return np.datetime64(pd.Timestamp(match.group(1)), 'ns')
|
|
93
|
+
raise ValueError(f'no acquisition time found in {path}')
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _open_levels(path):
|
|
97
|
+
"""Resolution levels (full resolution first) of a product, as lazy xarray.Datasets."""
|
|
98
|
+
from .driver import open_zarr_image, zarr_levels, open_hgrs_l2a
|
|
99
|
+
from .subset import open_l2a
|
|
100
|
+
|
|
101
|
+
if is_zarr(path):
|
|
102
|
+
return [open_zarr_image(path, level=level) for level in range(max(len(zarr_levels(path)), 1))]
|
|
103
|
+
if is_hgrs(path):
|
|
104
|
+
return [open_hgrs_l2a(path)[0]]
|
|
105
|
+
if os.path.isdir(path):
|
|
106
|
+
return [open_l2a(path)[0]]
|
|
107
|
+
return [xr.open_dataset(path, decode_coords='all', chunks={})]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _is_l2a(ds):
|
|
111
|
+
return 'Rrs' in ds and 'wl' in ds['Rrs'].dims
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _parameters(ds, dims=('y', 'x')):
|
|
115
|
+
"""Variables of an L2B product (or datacube) displayed by ViewParam: float maps."""
|
|
116
|
+
names = [name for name, var in ds.data_vars.items()
|
|
117
|
+
if set(var.dims) == set(dims) and var.dtype.kind == 'f' and name not in NOT_PARAMETERS]
|
|
118
|
+
|
|
119
|
+
def rank(name):
|
|
120
|
+
return next((i for i, prefix in enumerate(PARAMETER_ORDER) if name.startswith(prefix)), len(PARAMETER_ORDER))
|
|
121
|
+
|
|
122
|
+
return sorted(names, key=rank)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _layers(ds, kind, params=None, mask_invalid=True):
|
|
126
|
+
"""Layers displayed by the viewers: Rrs (L2A) or the parameters (L2B), masked with ``mask != 0``."""
|
|
127
|
+
if kind == 'L2A':
|
|
128
|
+
return ds['Rrs']
|
|
129
|
+
layers = ds[params]
|
|
130
|
+
if mask_invalid and 'mask' in ds:
|
|
131
|
+
layers = layers.where(ds['mask'] == 0)
|
|
132
|
+
return layers
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def open_products(paths, mask_invalid=True, min_size=512):
|
|
136
|
+
"""
|
|
137
|
+
Open products for the viewers, lazily.
|
|
138
|
+
|
|
139
|
+
The products can be on different grids (e.g. subsets of several tiles), but must share the same CRS.
|
|
140
|
+
|
|
141
|
+
:param paths: path, glob pattern, or list of them (see :func:`expand_paths`): L2A or L2B products
|
|
142
|
+
(one image each, all of the same level), or a single NetCDF datacube with a ``time``
|
|
143
|
+
dimension
|
|
144
|
+
:param mask_invalid: L2B: hide the pixels with ``mask != 0`` (not valid water pixels)
|
|
145
|
+
:param min_size: size (pixels) of the coarsest level computed for the products without pyramid
|
|
146
|
+
:return: (data, kind, params, title): :class:`~grstbx.visual.Multiscale`, ``'L2A'`` or ``'L2B'``,
|
|
147
|
+
displayed parameters (L2B, None for L2A), Markdown title of the viewer
|
|
148
|
+
"""
|
|
149
|
+
from .visual import Multiscale
|
|
150
|
+
|
|
151
|
+
paths = expand_paths(paths)
|
|
152
|
+
if not paths:
|
|
153
|
+
raise ValueError('no product given')
|
|
154
|
+
|
|
155
|
+
# NetCDF datacube: one file with a time dimension
|
|
156
|
+
if len(paths) == 1 and not is_zarr(paths[0]) and os.path.isfile(paths[0]) and not is_hgrs(paths[0]):
|
|
157
|
+
cube = xr.open_dataset(paths[0], decode_coords='all', chunks={})
|
|
158
|
+
if 'time' in cube.dims and cube.sizes['time'] > 1:
|
|
159
|
+
cube = cube.sortby('time')
|
|
160
|
+
kind = 'L2A' if _is_l2a(cube) else 'L2B'
|
|
161
|
+
params = None if kind == 'L2A' else _parameters(cube, dims=('time', 'y', 'x'))
|
|
162
|
+
if kind == 'L2B' and not params:
|
|
163
|
+
raise ValueError(f'no parameter to display in {paths[0]}')
|
|
164
|
+
layers = _layers(cube, kind, params, mask_invalid)
|
|
165
|
+
if layers.nbytes < MAX_IN_MEMORY:
|
|
166
|
+
layers = layers.load()
|
|
167
|
+
data = Multiscale.from_xarray(layers, min_size=min_size)
|
|
168
|
+
data.scheduler = 'synchronous' # dask scheduler of the exports (NetCDF)
|
|
169
|
+
title = f'## {os.path.basename(paths[0])}\n{kind} datacube, {cube.sizes["time"]} dates'
|
|
170
|
+
return data, kind, params, title
|
|
171
|
+
|
|
172
|
+
products = [(path, _open_levels(path)) for path in paths]
|
|
173
|
+
kinds = {'L2A' if _is_l2a(levels[0]) else 'L2B' for _, levels in products}
|
|
174
|
+
if len(kinds) > 1:
|
|
175
|
+
raise ValueError('L2A and L2B products cannot be displayed together')
|
|
176
|
+
kind = kinds.pop()
|
|
177
|
+
|
|
178
|
+
params = None
|
|
179
|
+
if kind == 'L2B':
|
|
180
|
+
# parameters common to all the products
|
|
181
|
+
params = _parameters(products[0][1][0])
|
|
182
|
+
for _, levels in products[1:]:
|
|
183
|
+
names = set(_parameters(levels[0]))
|
|
184
|
+
params = [param for param in params if param in names]
|
|
185
|
+
if not params:
|
|
186
|
+
raise ValueError('no parameter common to all the products')
|
|
187
|
+
|
|
188
|
+
crs = {levels[0].rio.crs for _, levels in products}
|
|
189
|
+
if len(crs) > 1:
|
|
190
|
+
raise ValueError(f'the products are in different CRS ({", ".join(map(str, crs))}): open the products '
|
|
191
|
+
'of one UTM zone at a time')
|
|
192
|
+
|
|
193
|
+
times = [_image_time(levels[0], path) for path, levels in products]
|
|
194
|
+
images = [[_layers(level, kind, params, mask_invalid) for level in levels] for _, levels in products]
|
|
195
|
+
order = np.argsort(times)
|
|
196
|
+
data = Multiscale([images[i] for i in order], np.array(times)[order], min_size=min_size)
|
|
197
|
+
data.scheduler = None if all(is_zarr(path) for path in paths) else 'synchronous'
|
|
198
|
+
|
|
199
|
+
if len(paths) == 1:
|
|
200
|
+
level = 'hGRS L2A' if is_hgrs(paths[0]) else f'GRS {kind}'
|
|
201
|
+
title = f'## {os.path.basename(paths[0])}\n{level}'
|
|
202
|
+
else:
|
|
203
|
+
dates = pd.DatetimeIndex(data.times)
|
|
204
|
+
title = f'## {len(paths)} {kind} products\nfrom {dates[0].date()} to {dates[-1].date()}'
|
|
205
|
+
return data, kind, params, title
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# -- exports ------------------------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
def _is_spectral(viewer):
|
|
211
|
+
return hasattr(viewer, 'wls')
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _as_layers(obj, viewer):
|
|
215
|
+
"""(labels, DataArray with a 'layer' first dimension) of an image or pixel of the viewer."""
|
|
216
|
+
if _is_spectral(viewer):
|
|
217
|
+
da = obj.sel(wl=list(viewer.wls)).rename(wl='layer')
|
|
218
|
+
return [f'Rrs_{wl:g}' for wl in viewer.wls], da
|
|
219
|
+
return list(viewer.params), obj[list(viewer.params)].to_array('layer')
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _compute(objects, viewer):
|
|
223
|
+
"""
|
|
224
|
+
Compute lazy xarray objects together, in a single dask graph. NetCDF (HDF5) reads are serialized by a
|
|
225
|
+
lock: they are faster on a single thread (see :class:`~grstbx.driver.L2grs`).
|
|
226
|
+
"""
|
|
227
|
+
import dask
|
|
228
|
+
|
|
229
|
+
with warnings.catch_warnings(): # all-NaN pixels or polygons
|
|
230
|
+
warnings.simplefilter('ignore', RuntimeWarning)
|
|
231
|
+
return dask.compute(*objects, scheduler=getattr(viewer.data, 'scheduler', None))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def points_values(viewer):
|
|
235
|
+
"""
|
|
236
|
+
Full-resolution values of the points of the map, for all the dates of the viewer: spectra
|
|
237
|
+
(:class:`~grstbx.visual.ViewSpectral`) or parameters (:class:`~grstbx.visual.ViewParam`).
|
|
238
|
+
|
|
239
|
+
:return: pandas.DataFrame, one row per point and date, one column per wavelength or parameter
|
|
240
|
+
"""
|
|
241
|
+
points = viewer.get_points()
|
|
242
|
+
native = points.to_crs(viewer.data.crs)
|
|
243
|
+
lazy, rows = [], []
|
|
244
|
+
for ipoint, (point, xy) in enumerate(zip(points.geometry, native.geometry)):
|
|
245
|
+
for date, idate in zip(viewer.datetimes, viewer.indexes):
|
|
246
|
+
xmin, ymin, xmax, ymax = viewer.data.bounds(idate)
|
|
247
|
+
if not (xmin <= xy.x <= xmax and ymin <= xy.y <= ymax): # point outside the image
|
|
248
|
+
continue
|
|
249
|
+
pixel = viewer.data.images[idate][0].sel(x=xy.x, y=xy.y, method='nearest')
|
|
250
|
+
labels, layers = _as_layers(pixel, viewer)
|
|
251
|
+
lazy.append(layers)
|
|
252
|
+
rows.append({'point': ipoint, 'lon': point.x, 'lat': point.y, 'time': date})
|
|
253
|
+
for row, values in zip(rows, _compute(lazy, viewer)):
|
|
254
|
+
row.update({label: float(v) for label, v in zip(labels, values.values)})
|
|
255
|
+
return pd.DataFrame(rows)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def polygons_statistics(viewer):
|
|
260
|
+
"""
|
|
261
|
+
Statistics of the full-resolution valid pixels within each polygon of the map, for all the dates of
|
|
262
|
+
the viewer: Rrs (:class:`~grstbx.visual.ViewSpectral`) or parameters (:class:`~grstbx.visual.ViewParam`).
|
|
263
|
+
|
|
264
|
+
:return: pandas.DataFrame, one row per polygon, date and statistic (median, mean, std, q05, q25, q75,
|
|
265
|
+
q95, count), one column per wavelength or parameter
|
|
266
|
+
"""
|
|
267
|
+
from rasterio.features import geometry_mask
|
|
268
|
+
|
|
269
|
+
n_poly = len((viewer.aoi_stream.data or {}).get('xs', []))
|
|
270
|
+
lazy, keys, masks = [], [], {}
|
|
271
|
+
for ipoly in range(n_poly):
|
|
272
|
+
aoi = viewer.get_geom(crs=viewer.data.crs, index=ipoly)
|
|
273
|
+
xmin, ymin, xmax, ymax = aoi.total_bounds
|
|
274
|
+
for date, idate in zip(viewer.datetimes, viewer.indexes):
|
|
275
|
+
image = viewer.data.images[idate][0]
|
|
276
|
+
ys = image.y.values
|
|
277
|
+
# bounding box of the polygon (lazy slicing)
|
|
278
|
+
image = image.sel(x=slice(xmin, xmax), y=slice(ymax, ymin) if ys[0] > ys[-1] else slice(ymin, ymax))
|
|
279
|
+
if image.sizes['x'] == 0 or image.sizes['y'] == 0: # polygon outside the image
|
|
280
|
+
continue
|
|
281
|
+
# pixels inside the polygon; the mask is computed once per grid (e.g. for all the dates of a
|
|
282
|
+
# datacube)
|
|
283
|
+
grid = (ipoly, image.sizes['y'], image.sizes['x'], tuple(image.rio.transform()))
|
|
284
|
+
if grid not in masks:
|
|
285
|
+
masks[grid] = xr.DataArray(
|
|
286
|
+
geometry_mask(aoi.geometry.values, out_shape=grid[1:3], transform=image.rio.transform(),
|
|
287
|
+
invert=True), dims=('y', 'x'), coords={'y': image.y, 'x': image.x})
|
|
288
|
+
labels, layers = _as_layers(image.where(masks[grid]), viewer)
|
|
289
|
+
lazy.append(layers)
|
|
290
|
+
keys.append((ipoly, date))
|
|
291
|
+
|
|
292
|
+
rows = []
|
|
293
|
+
for (ipoly, date), layers in zip(keys, _compute(lazy, viewer)):
|
|
294
|
+
values = layers.values.reshape(layers.shape[0], -1)
|
|
295
|
+
values = values[:, np.isfinite(values).any(axis=0)]
|
|
296
|
+
if values.shape[1] == 0: # no valid pixel
|
|
297
|
+
continue
|
|
298
|
+
with warnings.catch_warnings(): # all-NaN layers
|
|
299
|
+
warnings.simplefilter('ignore', RuntimeWarning)
|
|
300
|
+
stats = {'median': np.nanmedian(values, axis=1), 'mean': np.nanmean(values, axis=1),
|
|
301
|
+
'std': np.nanstd(values, axis=1), 'count': np.isfinite(values).sum(axis=1)}
|
|
302
|
+
for q in (0.05, 0.25, 0.75, 0.95):
|
|
303
|
+
stats[f'q{q * 100:02.0f}'] = np.nanquantile(values, q, axis=1)
|
|
304
|
+
for stat, column in stats.items():
|
|
305
|
+
row = {'polygon': ipoly, 'time': date, 'statistic': stat}
|
|
306
|
+
row.update({label: float(v) for label, v in zip(labels, column)})
|
|
307
|
+
rows.append(row)
|
|
308
|
+
return pd.DataFrame(rows)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _csv(df):
|
|
312
|
+
buffer = io.StringIO()
|
|
313
|
+
df.to_csv(buffer, index=False)
|
|
314
|
+
buffer.seek(0)
|
|
315
|
+
return buffer
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _geojson(gdf):
|
|
319
|
+
return io.StringIO(gdf.to_json())
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _exports(viewer):
|
|
323
|
+
"""Download buttons of the points, polygons, values and statistics drawn on the map."""
|
|
324
|
+
import geopandas as gpd
|
|
325
|
+
|
|
326
|
+
def polygons():
|
|
327
|
+
n_poly = len((viewer.aoi_stream.data or {}).get('xs', []))
|
|
328
|
+
if n_poly == 0:
|
|
329
|
+
return gpd.GeoDataFrame(geometry=[], crs=4326)
|
|
330
|
+
return pd.concat([viewer.get_geom(index=i) for i in range(n_poly)], ignore_index=True)
|
|
331
|
+
|
|
332
|
+
values = 'Spectra' if _is_spectral(viewer) else 'Parameters'
|
|
333
|
+
buttons = [
|
|
334
|
+
pn.widgets.FileDownload(callback=lambda: _csv(points_values(viewer)), filename='points_values.csv',
|
|
335
|
+
label=f'{values} of the points (CSV)', button_type='primary'),
|
|
336
|
+
pn.widgets.FileDownload(callback=lambda: _csv(polygons_statistics(viewer)),
|
|
337
|
+
filename='polygons_statistics.csv',
|
|
338
|
+
label='Statistics of the polygons (CSV)', button_type='primary'),
|
|
339
|
+
pn.widgets.FileDownload(callback=lambda: _geojson(viewer.get_points()), filename='points.geojson',
|
|
340
|
+
label='Points (GeoJSON)'),
|
|
341
|
+
pn.widgets.FileDownload(callback=lambda: _geojson(polygons()), filename='polygons.geojson',
|
|
342
|
+
label='Polygons (GeoJSON)'),
|
|
343
|
+
]
|
|
344
|
+
return pn.Column(
|
|
345
|
+
'### Export',
|
|
346
|
+
f'Full-resolution {values.lower()} of the points and statistics of the polygons drawn on the map, for '
|
|
347
|
+
f'all the dates ({len(viewer.indexes)}). The polygon statistics read all the pixels of the polygons: '
|
|
348
|
+
'large polygons or long time series take a while.',
|
|
349
|
+
pn.Row(*buttons))
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# -- dashboard ---------------------------------------------------------------------------------------
|
|
353
|
+
|
|
354
|
+
def _file_browser(root, products_input):
|
|
355
|
+
"""
|
|
356
|
+
Compact file browser of the server for the sidebar: a click on a folder opens it, a click on a
|
|
357
|
+
product (see :func:`is_product`) adds it to ``products_input``.
|
|
358
|
+
"""
|
|
359
|
+
folder = pn.widgets.TextInput(name='Folder', value=str(root))
|
|
360
|
+
entries = pn.widgets.MultiSelect(options={}, size=16, sizing_mode='stretch_width')
|
|
361
|
+
parent = pn.widgets.Button(name='Parent folder', icon='arrow-up', button_type='light')
|
|
362
|
+
add_all = pn.widgets.Button(name='Add all the products', icon='list-check', button_type='light')
|
|
363
|
+
|
|
364
|
+
def listing():
|
|
365
|
+
try:
|
|
366
|
+
names = sorted(n for n in os.listdir(folder.value) if not n.startswith('.'))
|
|
367
|
+
except OSError as exc:
|
|
368
|
+
pn.state.notifications.error(str(exc))
|
|
369
|
+
return [], []
|
|
370
|
+
paths = [os.path.join(folder.value, n) for n in names]
|
|
371
|
+
products = [p for p in paths if is_product(p)]
|
|
372
|
+
return [p for p in paths if os.path.isdir(p) and p not in products], products
|
|
373
|
+
|
|
374
|
+
def refresh(*events):
|
|
375
|
+
folders, products = listing()
|
|
376
|
+
options = {'\N{FILE FOLDER} ' + os.path.basename(p): p for p in folders}
|
|
377
|
+
options.update({'\N{GLOBE WITH MERIDIANS} ' + os.path.basename(p): p for p in products})
|
|
378
|
+
entries.options = options
|
|
379
|
+
|
|
380
|
+
def add(paths):
|
|
381
|
+
current = [line.strip() for line in products_input.value.splitlines() if line.strip()]
|
|
382
|
+
products_input.value = '\n'.join(current + [p for p in paths if p not in current])
|
|
383
|
+
|
|
384
|
+
def on_click(event):
|
|
385
|
+
if not event.new:
|
|
386
|
+
return
|
|
387
|
+
selected = event.new[0]
|
|
388
|
+
entries.value = []
|
|
389
|
+
if is_product(selected):
|
|
390
|
+
add([selected])
|
|
391
|
+
else:
|
|
392
|
+
folder.value = selected
|
|
393
|
+
|
|
394
|
+
def on_parent(event):
|
|
395
|
+
folder.value = os.path.dirname(folder.value.rstrip('/')) or '/'
|
|
396
|
+
|
|
397
|
+
folder.param.watch(refresh, 'value')
|
|
398
|
+
entries.param.watch(on_click, 'value')
|
|
399
|
+
parent.on_click(on_parent)
|
|
400
|
+
add_all.on_click(lambda event: add(listing()[1]))
|
|
401
|
+
refresh()
|
|
402
|
+
return pn.Column(folder, pn.Row(parent, add_all), entries,
|
|
403
|
+
pn.pane.Markdown('\N{GLOBE WITH MERIDIANS}: product (click to add it), '
|
|
404
|
+
'\N{FILE FOLDER}: folder (click to open it)'),
|
|
405
|
+
sizing_mode='stretch_width')
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def create_app(path=None, root=None, reproject=True, basemap=None, cmap=None, width=1000, height=700):
|
|
409
|
+
"""
|
|
410
|
+
Build the dashboard (one per browser session).
|
|
411
|
+
|
|
412
|
+
:param path: product(s) opened at start-up (optional): path, glob pattern or list of them
|
|
413
|
+
:param root: directory of the file browser of the sidebar (default: directory of the first product or
|
|
414
|
+
current directory)
|
|
415
|
+
:param reproject: display over basemaps (web Mercator)
|
|
416
|
+
:param basemap: initial basemap (see :data:`grstbx.visual.BASEMAPS`)
|
|
417
|
+
:param cmap: initial colormap (see :class:`~grstbx.visual.ViewSpectral`)
|
|
418
|
+
:param width: width of the map (pixels)
|
|
419
|
+
:param height: height of the map (pixels)
|
|
420
|
+
:return: panel template
|
|
421
|
+
"""
|
|
422
|
+
from . import visual
|
|
423
|
+
|
|
424
|
+
pn.extension(notifications=True)
|
|
425
|
+
paths = [path] if isinstance(path, (str, os.PathLike)) else list(path or [])
|
|
426
|
+
if root is None:
|
|
427
|
+
first = str(paths[0]).rstrip('/') if paths else ''
|
|
428
|
+
root = os.path.dirname(first.split('*')[0]) if first else os.getcwd()
|
|
429
|
+
if not os.path.isdir(root):
|
|
430
|
+
root = os.getcwd()
|
|
431
|
+
|
|
432
|
+
products_input = pn.widgets.TextAreaInput(
|
|
433
|
+
name='Products', value='\n'.join(map(str, paths)), height=110, sizing_mode='stretch_width',
|
|
434
|
+
placeholder='One path or glob pattern per line, e.g.\n/data/L2A/product.zarr\n/data/L2B/BEFR/2024/*/*/*.nc')
|
|
435
|
+
browser = _file_browser(root, products_input)
|
|
436
|
+
clear_button = pn.widgets.Button(name='Clear', button_type='light')
|
|
437
|
+
reproject_box = pn.widgets.Checkbox(name='Basemap (reproject to web Mercator)', value=reproject)
|
|
438
|
+
basemap_select = pn.widgets.Select(name='Basemap', options=list(visual.BASEMAPS),
|
|
439
|
+
value=basemap or visual.DEFAULT_BASEMAP)
|
|
440
|
+
cmap_select = pn.widgets.Select(name='Colormap', options=list(visual.RRS_COLORMAPS),
|
|
441
|
+
value=next((label for label, name in visual.RRS_COLORMAPS.items()
|
|
442
|
+
if cmap in (label, name)), list(visual.RRS_COLORMAPS)[0]))
|
|
443
|
+
rrs_range = pn.widgets.EditableRangeSlider(name='L2A: initial Rrs range (sr-1)', start=0, end=0.1,
|
|
444
|
+
value=(0, 0.02), step=0.001)
|
|
445
|
+
param_range = pn.widgets.EditableRangeSlider(name='L2B: initial range of the parameters', start=0, end=100,
|
|
446
|
+
value=(0, 10), step=0.1)
|
|
447
|
+
mask_box = pn.widgets.Checkbox(name='L2B: hide the invalid pixels (mask != 0)', value=True)
|
|
448
|
+
load_button = pn.widgets.Button(name='Open', button_type='primary')
|
|
449
|
+
# controls of the opened viewer (filled when the products are opened)
|
|
450
|
+
contrast_card = pn.Card(title='Contrast and brightness', visible=False, sizing_mode='stretch_width')
|
|
451
|
+
rgb_card = pn.Card(title='RGB composite (L2A)', visible=False, sizing_mode='stretch_width')
|
|
452
|
+
|
|
453
|
+
main = pn.Column(sizing_mode='stretch_width')
|
|
454
|
+
intro = pn.pane.Markdown(
|
|
455
|
+
'## Open GRS products\n\n'
|
|
456
|
+
f'Enter the paths of {ACCEPTED} on the server, one path or glob pattern per line, or add them with '
|
|
457
|
+
'the file browser, then click **Open**. Several products make a time series: they can be on different '
|
|
458
|
+
'grids, but must be in the same CRS (UTM zone).\n\n'
|
|
459
|
+
'- **L2A** products are displayed with the spectral viewer: pick the band or the true-colour '
|
|
460
|
+
'composite, hover the map to plot the full-resolution spectrum under the mouse (dashed line), '
|
|
461
|
+
'add points with the **Point Draw** tool to plot their spectra;\n'
|
|
462
|
+
'- **L2B** products are displayed with the parameter viewer: pick the parameter and the date, hover '
|
|
463
|
+
'the map to plot the time series under the mouse, add points to plot their time series.\n\n'
|
|
464
|
+
'Pan and zoom the map: only the pixels displayed are read, from the Zarr pyramid levels. Draw '
|
|
465
|
+
'polygons with the **Polygon Draw** tool (double-click to start and to close), and download the '
|
|
466
|
+
'values of the points and the statistics of the polygons below the map.')
|
|
467
|
+
main.append(intro)
|
|
468
|
+
|
|
469
|
+
def on_load(event):
|
|
470
|
+
if not products_input.value.strip():
|
|
471
|
+
pn.state.notifications.warning('Enter or add products first.')
|
|
472
|
+
return
|
|
473
|
+
load_button.loading = True
|
|
474
|
+
main.loading = True
|
|
475
|
+
try:
|
|
476
|
+
data, kind, params, title = open_products(products_input.value, mask_invalid=mask_box.value)
|
|
477
|
+
options = dict(reproject=reproject_box.value, basemap=basemap_select.value, cmap=cmap_select.value,
|
|
478
|
+
width=width, height=height, title=title)
|
|
479
|
+
if kind == 'L2A':
|
|
480
|
+
viewer = visual.ViewSpectral(data, minmaxvalues=tuple(rrs_range.value),
|
|
481
|
+
minmax=(0, max(0.1, rrs_range.value[1])), **options)
|
|
482
|
+
else:
|
|
483
|
+
viewer = visual.ViewParam(data, params=params, minmaxvalues=tuple(param_range.value),
|
|
484
|
+
minmax=(0, max(100, param_range.value[1])), **options)
|
|
485
|
+
main.objects = [viewer.visu(controls=False), _exports(viewer)]
|
|
486
|
+
contrast_card.objects = [viewer.display_controls()]
|
|
487
|
+
contrast_card.visible = True
|
|
488
|
+
rgb_card.objects = [viewer.rgb_controls()] if kind == 'L2A' else []
|
|
489
|
+
rgb_card.visible = kind == 'L2A'
|
|
490
|
+
except Exception as exc:
|
|
491
|
+
pn.state.notifications.error(f'Cannot open the products: {exc}', duration=0)
|
|
492
|
+
main.objects = [intro]
|
|
493
|
+
contrast_card.visible = rgb_card.visible = False
|
|
494
|
+
finally:
|
|
495
|
+
load_button.loading = False
|
|
496
|
+
main.loading = False
|
|
497
|
+
|
|
498
|
+
load_button.on_click(on_load)
|
|
499
|
+
clear_button.on_click(lambda event: setattr(products_input, 'value', ''))
|
|
500
|
+
if paths:
|
|
501
|
+
pn.state.onload(lambda: on_load(None))
|
|
502
|
+
|
|
503
|
+
sidebar = [
|
|
504
|
+
products_input,
|
|
505
|
+
pn.Row(load_button, clear_button),
|
|
506
|
+
pn.Card(browser, title='File browser', collapsed=bool(paths), sizing_mode='stretch_width'),
|
|
507
|
+
pn.Card(reproject_box, basemap_select, cmap_select, rrs_range, param_range, mask_box,
|
|
508
|
+
title='Options (applied at opening)', collapsed=True, sizing_mode='stretch_width'),
|
|
509
|
+
contrast_card,
|
|
510
|
+
rgb_card,
|
|
511
|
+
]
|
|
512
|
+
return pn.template.FastListTemplate(title='grstbx viewer', sidebar=sidebar, main=[main],
|
|
513
|
+
sidebar_width=380, header_background='#1f4e79')
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def serve(path=None, root=None, port=5006, address=None, show=True, allow_websocket_origin=None, **kwargs):
|
|
517
|
+
"""
|
|
518
|
+
Serve the dashboard (blocking).
|
|
519
|
+
|
|
520
|
+
:param path: product(s) opened at start-up (optional): path, glob pattern or list of them
|
|
521
|
+
:param root: directory of the file browser
|
|
522
|
+
:param port: port of the server
|
|
523
|
+
:param address: address to listen on (e.g. '0.0.0.0' to accept connections from other computers)
|
|
524
|
+
:param show: open the dashboard in the web browser
|
|
525
|
+
:param allow_websocket_origin: list of hosts allowed to connect (e.g. ['myserver:5006']), added to the
|
|
526
|
+
names of the local machine (localhost, 127.0.0.1, host name, address)
|
|
527
|
+
:param kwargs: other arguments of :func:`create_app`
|
|
528
|
+
"""
|
|
529
|
+
pn.serve(lambda: create_app(path=path, root=root, **kwargs), port=port, address=address, show=show,
|
|
530
|
+
websocket_origin=_websocket_origins(port, address, allow_websocket_origin),
|
|
531
|
+
title='grstbx viewer')
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _websocket_origins(port, address=None, extra=None):
|
|
535
|
+
"""
|
|
536
|
+
Hosts allowed to open the websocket of the dashboard. Bokeh allows only ``localhost:<port>`` by default,
|
|
537
|
+
so that a page opened as ``127.0.0.1:<port>`` (or with the host name) stays empty.
|
|
538
|
+
"""
|
|
539
|
+
import socket
|
|
540
|
+
|
|
541
|
+
hosts = {'localhost', '127.0.0.1', socket.gethostname(), socket.getfqdn()}
|
|
542
|
+
if address and address not in ('0.0.0.0', '::'):
|
|
543
|
+
hosts.add(address)
|
|
544
|
+
origins = sorted(f'{host}:{port}' for host in hosts if host)
|
|
545
|
+
return origins + [origin for origin in (extra or []) if origin not in origins]
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def main(argv=None):
|
|
549
|
+
"""Command line entry point: ``grstbx-viewer``."""
|
|
550
|
+
parser = argparse.ArgumentParser(prog='grstbx-viewer',
|
|
551
|
+
description='Web dashboard to explore GRS L2A and L2B products, '
|
|
552
|
+
'single images or time series.')
|
|
553
|
+
parser.add_argument('path', nargs='*',
|
|
554
|
+
help=f'products opened at start-up (paths or quoted glob patterns): {ACCEPTED}')
|
|
555
|
+
parser.add_argument('--root', help='directory of the file browser (default: directory of the first '
|
|
556
|
+
'product, or current directory)')
|
|
557
|
+
parser.add_argument('--port', type=int, default=5006, help='port of the server (default: 5006)')
|
|
558
|
+
parser.add_argument('--address', help="address to listen on, e.g. 0.0.0.0 to accept connections from other "
|
|
559
|
+
"computers (default: localhost)")
|
|
560
|
+
parser.add_argument('--allow-websocket-origin', action='append',
|
|
561
|
+
help='host[:port] allowed to connect, repeat for several hosts; localhost, 127.0.0.1 and '
|
|
562
|
+
'the host name of the machine are always allowed')
|
|
563
|
+
parser.add_argument('--no-show', action='store_true', help='do not open the web browser')
|
|
564
|
+
parser.add_argument('--no-reproject', action='store_true',
|
|
565
|
+
help='display in the projection of the images, without basemap')
|
|
566
|
+
parser.add_argument('--basemap', help='initial basemap, e.g. "CARTO Positron"')
|
|
567
|
+
parser.add_argument('--cmap', help="initial colormap, e.g. 'cmo.turbid'")
|
|
568
|
+
parser.add_argument('--stadia-api-key', help='API key of the Stadia Maps basemaps (or STADIA_API_KEY)')
|
|
569
|
+
args = parser.parse_args(argv)
|
|
570
|
+
|
|
571
|
+
if args.stadia_api_key:
|
|
572
|
+
from .visual import set_stadia_api_key
|
|
573
|
+
set_stadia_api_key(args.stadia_api_key)
|
|
574
|
+
serve(path=args.path, root=args.root, port=args.port, address=args.address, show=not args.no_show,
|
|
575
|
+
allow_websocket_origin=args.allow_websocket_origin, reproject=not args.no_reproject,
|
|
576
|
+
basemap=args.basemap, cmap=args.cmap)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
if __name__ == '__main__':
|
|
580
|
+
main()
|
|
Binary file
|