meteoraster 2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meteoraster/__init__.py +5 -0
- meteoraster/meteoraster.py +1267 -0
- meteoraster/resample.py +269 -0
- meteoraster/utils.py +359 -0
- meteoraster-2.5.dist-info/METADATA +419 -0
- meteoraster-2.5.dist-info/RECORD +10 -0
- meteoraster-2.5.dist-info/WHEEL +5 -0
- meteoraster-2.5.dist-info/licenses/LICENSE +202 -0
- meteoraster-2.5.dist-info/licenses/NOTICE +18 -0
- meteoraster-2.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1267 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import copy
|
|
3
|
+
import gc
|
|
4
|
+
import os
|
|
5
|
+
import warnings
|
|
6
|
+
import shutil
|
|
7
|
+
import tempfile
|
|
8
|
+
import numpy as np
|
|
9
|
+
import xarray as xr
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import seaborn as sns
|
|
13
|
+
import matplotlib.pyplot as plt
|
|
14
|
+
import xml.etree.ElementTree as et
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from osgeo import ogr
|
|
17
|
+
import scipy.interpolate as interp
|
|
18
|
+
import cartopy.crs as ccrs
|
|
19
|
+
import cartopy.feature as cf
|
|
20
|
+
import cartopy.io.shapereader as shpreader
|
|
21
|
+
|
|
22
|
+
class MeteoRaster(object):
|
|
23
|
+
'''
|
|
24
|
+
Custom class to handle meteorological raster files, including ensembles and forecasts
|
|
25
|
+
|
|
26
|
+
v2.5:
|
|
27
|
+
Published on PyPI - installs with `pip install meteoraster`. Releases are
|
|
28
|
+
built and uploaded automatically on tag push (GitHub Actions + PyPI
|
|
29
|
+
Trusted Publishing), and attached to the GitHub release.
|
|
30
|
+
Licensing made consistent: the LICENSE file carried GPL-3 text while the
|
|
31
|
+
package metadata declared Apache-2.0. Apache-2.0 is now the license
|
|
32
|
+
everywhere, declared as a PEP 639 SPDX expression, with a NOTICE file.
|
|
33
|
+
|
|
34
|
+
v2.4:
|
|
35
|
+
Packaging modernized (pyproject.toml, optional dependencies, h5netcdf and
|
|
36
|
+
pytest declared as dependencies).
|
|
37
|
+
Cleaned up deprecated numpy/pandas idioms (np.NaN -> np.nan,
|
|
38
|
+
np.product -> np.prod, removed redundant axis= arguments) for
|
|
39
|
+
forward-compatibility with numpy 2 and pandas 3.
|
|
40
|
+
|
|
41
|
+
v2.3:
|
|
42
|
+
Fixed a bug in the groupByMatrix function where the coverage was not updated to reflect missing data, which could lead to incorrect aggregations. The coverage is now set to 0 for pixels where data is not valid before performing the aggregation.
|
|
43
|
+
|
|
44
|
+
v2.2:
|
|
45
|
+
Specifies the saving engine as 'h5netcdf'
|
|
46
|
+
|
|
47
|
+
v2.1:
|
|
48
|
+
Updated to VSCode
|
|
49
|
+
Updated to drop pickle and use cross-platfrom alternatives instead
|
|
50
|
+
|
|
51
|
+
Added testing
|
|
52
|
+
Removed support for reading multiple data formats
|
|
53
|
+
A "latest" version is compiled and published
|
|
54
|
+
|
|
55
|
+
Untested support for pd.DateOffset
|
|
56
|
+
'''
|
|
57
|
+
|
|
58
|
+
VERSION = '2.5'
|
|
59
|
+
VERBOSE = True
|
|
60
|
+
ENSEMBLEMEMBERpOSITION = 1
|
|
61
|
+
|
|
62
|
+
ENGINE = 'h5netcdf'
|
|
63
|
+
|
|
64
|
+
def __init__(self, data,
|
|
65
|
+
latitudes=None, longitudes=None, production_datetime=None, leadtimes=None,
|
|
66
|
+
units='unknown', variable='unknown', da_attrs={}, ds_attrs={},
|
|
67
|
+
verbose=None,
|
|
68
|
+
):
|
|
69
|
+
'''
|
|
70
|
+
data: 5-D numpy array [production_datetime, ensemble_member, leadtime, latitude, longitude]
|
|
71
|
+
production dates: 1-D numpy array
|
|
72
|
+
ensemble members: 1-D numpy array
|
|
73
|
+
leadtimes: 1-D numpy array
|
|
74
|
+
latitudes: 1-D numpy array (center)
|
|
75
|
+
longitudes: 1-D numpy array (center)
|
|
76
|
+
|
|
77
|
+
alternatively, data can be a dict with the previous fields
|
|
78
|
+
'''
|
|
79
|
+
|
|
80
|
+
if verbose is None:
|
|
81
|
+
self.verbose = self.VERBOSE
|
|
82
|
+
else:
|
|
83
|
+
self.verbose = verbose
|
|
84
|
+
|
|
85
|
+
self.units = units
|
|
86
|
+
self.variable = variable
|
|
87
|
+
self.da_attrs = da_attrs
|
|
88
|
+
self.ds_attrs = ds_attrs
|
|
89
|
+
self._update_attrs()
|
|
90
|
+
|
|
91
|
+
if isinstance(data, dict):
|
|
92
|
+
self.data = data['data']
|
|
93
|
+
self.latitudes = data['latitudes']
|
|
94
|
+
self.longitudes = data['longitudes']
|
|
95
|
+
self.production_datetime = data['production_datetime']
|
|
96
|
+
self.leadtimes = data['leadtimes']
|
|
97
|
+
else:
|
|
98
|
+
self.data = data
|
|
99
|
+
|
|
100
|
+
if not latitudes is None:
|
|
101
|
+
self.latitudes = latitudes
|
|
102
|
+
if not longitudes is None:
|
|
103
|
+
self.longitudes = longitudes
|
|
104
|
+
if not production_datetime is None:
|
|
105
|
+
self.production_datetime = production_datetime
|
|
106
|
+
if not leadtimes is None:
|
|
107
|
+
self.leadtimes = leadtimes
|
|
108
|
+
|
|
109
|
+
# Fix Geomery
|
|
110
|
+
#Greenwish
|
|
111
|
+
if self.longitudes.ndim==1:
|
|
112
|
+
if max(self.longitudes)>180:
|
|
113
|
+
self._fixStartAtGreenwich()
|
|
114
|
+
|
|
115
|
+
# Process geometry so that lats and lons are stored in 2D
|
|
116
|
+
if self.longitudes.ndim==1 and self.latitudes.ndim==1:
|
|
117
|
+
self.longitudes, self.latitudes = np.meshgrid(self.longitudes, self.latitudes)
|
|
118
|
+
|
|
119
|
+
# Flip latitudes (if required)
|
|
120
|
+
if self.latitudes[0, 0]<self.latitudes[1, 0]:
|
|
121
|
+
self._flipLatitude()
|
|
122
|
+
|
|
123
|
+
def _update_attrs(self):
|
|
124
|
+
'''
|
|
125
|
+
Docstring for _update_attrs
|
|
126
|
+
|
|
127
|
+
:param self: Description
|
|
128
|
+
'''
|
|
129
|
+
|
|
130
|
+
if self.units == 'unknown':
|
|
131
|
+
if 'units' in self.da_attrs.keys():
|
|
132
|
+
self.units = self.da_attrs['units']
|
|
133
|
+
else:
|
|
134
|
+
self.da_attrs['units'] = self.units
|
|
135
|
+
|
|
136
|
+
if self.variable == 'unknown':
|
|
137
|
+
if 'variable' in self.da_attrs.keys():
|
|
138
|
+
self.variable = self.da_attrs['variable']
|
|
139
|
+
else:
|
|
140
|
+
self.da_attrs['variable'] = self.variable
|
|
141
|
+
|
|
142
|
+
def create_plot(self, central_longitude, projection_kwargs={}, figsize=None, projection_fun=ccrs.PlateCarree, crs=ccrs.PlateCarree(), extent=None):
|
|
143
|
+
'''
|
|
144
|
+
extent [west, east, south, north]
|
|
145
|
+
'''
|
|
146
|
+
|
|
147
|
+
plt.figure(figsize=figsize)
|
|
148
|
+
projection = projection_fun(central_longitude=central_longitude, **projection_kwargs)
|
|
149
|
+
ax = plt.axes(projection=projection)
|
|
150
|
+
if not isinstance(extent, type(None)):
|
|
151
|
+
ax.set_extent(extent, crs=crs)
|
|
152
|
+
return ax
|
|
153
|
+
|
|
154
|
+
def add_shapefile(self, ax, shapefile_path, crs=ccrs.PlateCarree(), **kwargs):
|
|
155
|
+
'''
|
|
156
|
+
|
|
157
|
+
'''
|
|
158
|
+
|
|
159
|
+
ax.add_geometries(shpreader.Reader(shapefile_path).geometries(), crs=crs, **kwargs)
|
|
160
|
+
|
|
161
|
+
def add_shape(self, ax, path, color='black', facecolor='none', linewidth=1, crs=ccrs.PlateCarree()):
|
|
162
|
+
'''
|
|
163
|
+
|
|
164
|
+
'''
|
|
165
|
+
reader = shpreader.Reader(path)
|
|
166
|
+
|
|
167
|
+
for record in reader.records():
|
|
168
|
+
geometry = record.geometry
|
|
169
|
+
ax.add_geometries([geometry], crs=crs, edgecolor=color, facecolor=facecolor, linewidth=linewidth)
|
|
170
|
+
plt.show(block=False)
|
|
171
|
+
|
|
172
|
+
def plot_mean(self, ax=None, xarray=None, block=False, multiplier=1, coastline=False, borders=False, colorbar=True,
|
|
173
|
+
colorbar_label=None, cmap='viridis', central_longitude=None, central_latitude=None, *args, **kwargs):
|
|
174
|
+
'''
|
|
175
|
+
Plots the mean behavior for the full time series
|
|
176
|
+
'''
|
|
177
|
+
|
|
178
|
+
if isinstance(ax, type(None)):
|
|
179
|
+
ax = self.create_plot(central_longitude)
|
|
180
|
+
|
|
181
|
+
if isinstance(xarray, type(None)):
|
|
182
|
+
xarray = self.to_xarray()
|
|
183
|
+
xarray = xarray.mean(dim=['production_datetime', 'ensemble_member', 'leadtime'])*multiplier
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
plot = xarray.plot.pcolormesh(ax=ax, transform=ccrs.PlateCarree(), x="lon", y="lat", add_colorbar=False, cmap=cmap, *args, **kwargs)
|
|
187
|
+
if colorbar:
|
|
188
|
+
if colorbar_label is None:
|
|
189
|
+
colorbar_label = self.units
|
|
190
|
+
cbar = plt.colorbar(plot, orientation='vertical', label=colorbar_label)
|
|
191
|
+
else:
|
|
192
|
+
cbar = None
|
|
193
|
+
|
|
194
|
+
if coastline:
|
|
195
|
+
ax.add_feature(cf.COASTLINE)
|
|
196
|
+
if borders:
|
|
197
|
+
ax.add_feature(cf.BORDERS)
|
|
198
|
+
|
|
199
|
+
plt.show(block=block)
|
|
200
|
+
|
|
201
|
+
return ax, cbar
|
|
202
|
+
|
|
203
|
+
def plot_mean_projected(self, ax=None, block=False, multiplier=1, *args, **kwargs):
|
|
204
|
+
'''
|
|
205
|
+
Plots the mean behavior for the full time series using the true coordinates
|
|
206
|
+
'''
|
|
207
|
+
|
|
208
|
+
sns.color_palette('viridis')
|
|
209
|
+
|
|
210
|
+
tmp = np.nanmean(self.data, axis=(0,1,2))*multiplier
|
|
211
|
+
|
|
212
|
+
if isinstance(ax, type(None)):
|
|
213
|
+
plt.figure()
|
|
214
|
+
|
|
215
|
+
plt.scatter(self.longitudes.ravel(), self.latitudes.ravel(), c=tmp.ravel(), *args, **kwargs)
|
|
216
|
+
plt.colorbar()
|
|
217
|
+
plt.show(block=block)
|
|
218
|
+
|
|
219
|
+
def plot_coordinates(self, ax=None, block=False, *args, **kwargs):
|
|
220
|
+
'''
|
|
221
|
+
Plots the available coordinates in the lat and lon matrixes
|
|
222
|
+
'''
|
|
223
|
+
|
|
224
|
+
if isinstance(ax, type(None)):
|
|
225
|
+
plt.figure()
|
|
226
|
+
|
|
227
|
+
plt.plot(self.longitudes.ravel(), self.latitudes.ravel(), 'k.', *args, **kwargs)
|
|
228
|
+
plt.show(block=block)
|
|
229
|
+
|
|
230
|
+
def plot_seasonal(self, lat, lon, window=31, leadtime=None, ax=None, block=False, multiplier=1, *args, **kwargs):
|
|
231
|
+
|
|
232
|
+
data = self.get_values_from_latlon(lat, lon)
|
|
233
|
+
if isinstance(leadtime, type(None)):
|
|
234
|
+
leadtime = self.leadtimes[0]
|
|
235
|
+
|
|
236
|
+
# Select specific leadtime columns (level 0 is leadtime)
|
|
237
|
+
data = data.xs(leadtime, level='leadtime', axis=1)
|
|
238
|
+
|
|
239
|
+
tmp = data.rolling(window=window, center=True).mean().dropna().stack().to_frame(name='values').reset_index()
|
|
240
|
+
|
|
241
|
+
# Use proper column name from index
|
|
242
|
+
idx_name = data.index.name if data.index.name else 'index'
|
|
243
|
+
if idx_name in tmp.columns:
|
|
244
|
+
tmp.rename(columns={idx_name: 'Production dates'}, inplace=True)
|
|
245
|
+
|
|
246
|
+
tmp.loc[:, 'Day of year'] = tmp.loc[:, 'Production dates'].dt.day_of_year
|
|
247
|
+
|
|
248
|
+
if isinstance(ax, type(None)):
|
|
249
|
+
plt.figure(figsize=(20,10))
|
|
250
|
+
ax = sns.lineplot(x='Day of year', y='values', data=tmp, errorbar=('pi', 95), ax=ax, *args, **kwargs)
|
|
251
|
+
|
|
252
|
+
return ax
|
|
253
|
+
|
|
254
|
+
def get_values_from_KML(self, kml, nameField=None, coverage_info=None, getCoverageInfo=False, elementwise=False):
|
|
255
|
+
'''
|
|
256
|
+
Returns a pandas table with averaged values
|
|
257
|
+
'''
|
|
258
|
+
|
|
259
|
+
if isinstance(coverage_info, type(None)):
|
|
260
|
+
cvr, columns, centroids = self.__coverageMatrixFromKML(kml, nameField)
|
|
261
|
+
coverage_info = (cvr, columns, centroids)
|
|
262
|
+
else:
|
|
263
|
+
cvr, columns, centroids = coverage_info
|
|
264
|
+
|
|
265
|
+
agg = self.__groupByMatrix(cvr, columns, elementwise=elementwise)
|
|
266
|
+
|
|
267
|
+
if getCoverageInfo:
|
|
268
|
+
return agg, centroids, coverage_info
|
|
269
|
+
else:
|
|
270
|
+
return agg, centroids
|
|
271
|
+
|
|
272
|
+
def get_quantiles_from_KML(self, kml, nameField=None, coverage_info=None, getCoverageInfo=False, resampling=None, precision=0.1, quantiles=[0.01, 0.1, 0.5, 0.9, 0.99]):
|
|
273
|
+
'''
|
|
274
|
+
Returns a pandas table with averaged values
|
|
275
|
+
'''
|
|
276
|
+
|
|
277
|
+
if isinstance(coverage_info, type(None)):
|
|
278
|
+
cvr, columns, centroids = self.__coverageMatrixFromKML(kml, nameField, normalize_by_area=False)
|
|
279
|
+
coverage_info = (cvr, columns, centroids)
|
|
280
|
+
else:
|
|
281
|
+
cvr, columns, centroids = coverage_info
|
|
282
|
+
|
|
283
|
+
agg = self.__groupByQuantile(cvr, columns, resampling, quantiles, precision)
|
|
284
|
+
|
|
285
|
+
if getCoverageInfo:
|
|
286
|
+
return agg, centroids, coverage_info
|
|
287
|
+
else:
|
|
288
|
+
return agg, centroids
|
|
289
|
+
|
|
290
|
+
def get_cropped(self, from_prod_date=dt.datetime(1900, 1, 1), to_prod_date=dt.datetime(2199, 12, 31),
|
|
291
|
+
from_lat=-90, to_lat=90, from_lon=-180, to_lon=180,
|
|
292
|
+
from_leadtime=None, to_leadtime=None):
|
|
293
|
+
'''
|
|
294
|
+
returns a MeteoRaster cropped in time and space
|
|
295
|
+
'''
|
|
296
|
+
|
|
297
|
+
self._diag('Cropping meteorology...', self.verbose)
|
|
298
|
+
|
|
299
|
+
date_idxs = np.arange(self.production_datetime.shape[0])[(self.production_datetime>=np.datetime64(from_prod_date)) & (self.production_datetime<=np.datetime64(to_prod_date))]
|
|
300
|
+
|
|
301
|
+
inside = np.zeros_like(self.latitudes).astype(bool)
|
|
302
|
+
inside[(np.round(self.longitudes,6)>=np.round(from_lon,6)) & (np.round(self.longitudes,6)<=np.round(to_lon,6)) & (np.round(self.latitudes,6)>=np.round(from_lat,6)) & (np.round(self.latitudes,6)<=np.round(to_lat,6))] = True
|
|
303
|
+
tmp = np.where(inside)
|
|
304
|
+
lat_idxs = np.arange(tmp[0].min(), tmp[0].max()+1)
|
|
305
|
+
lon_idxs = np.arange(tmp[1].min(), tmp[1].max()+1)
|
|
306
|
+
|
|
307
|
+
#=======================================================================
|
|
308
|
+
# lat_idxs = np.arange(self.latitudes.shape[0])[(self.latitudes>=from_lat) & (self.latitudes<=to_lat)]
|
|
309
|
+
# lon_idxs = np.arange(self.longitudes.shape[0])[(self.longitudes>=from_lon) & (self.longitudes<=to_lon)]
|
|
310
|
+
#=======================================================================
|
|
311
|
+
|
|
312
|
+
production_datetime = self.production_datetime[date_idxs]
|
|
313
|
+
latitudes = np.round(self.latitudes[lat_idxs, :],6)
|
|
314
|
+
latitudes = np.round(latitudes[:, lon_idxs],6)
|
|
315
|
+
longitudes = np.round(self.longitudes[lat_idxs, :],6)
|
|
316
|
+
longitudes = np.round(longitudes[:, lon_idxs],6)
|
|
317
|
+
data = self.data[date_idxs, :, :, :, :]
|
|
318
|
+
data = data[:, :, :, lat_idxs, :]
|
|
319
|
+
data = data[:, :, :, :, lon_idxs]
|
|
320
|
+
|
|
321
|
+
cropped = copy.deepcopy(self)
|
|
322
|
+
cropped.production_datetime = production_datetime
|
|
323
|
+
cropped.latitudes = latitudes
|
|
324
|
+
cropped.longitudes = longitudes
|
|
325
|
+
|
|
326
|
+
if from_leadtime:
|
|
327
|
+
idxs = cropped.leadtimes>=from_leadtime
|
|
328
|
+
cropped.leadtimes = cropped.leadtimes[idxs]
|
|
329
|
+
data = data[:, :, idxs, :, :]
|
|
330
|
+
|
|
331
|
+
if to_leadtime:
|
|
332
|
+
idxs = cropped.leadtimes<=to_leadtime
|
|
333
|
+
cropped.leadtimes = cropped.leadtimes[idxs]
|
|
334
|
+
data = data[:, :, idxs, :, :]
|
|
335
|
+
|
|
336
|
+
cropped.data = data
|
|
337
|
+
|
|
338
|
+
self._diag(' Done.', self.verbose)
|
|
339
|
+
|
|
340
|
+
return cropped
|
|
341
|
+
|
|
342
|
+
def trim(self):
|
|
343
|
+
'''
|
|
344
|
+
|
|
345
|
+
'''
|
|
346
|
+
with warnings.catch_warnings():
|
|
347
|
+
warnings.simplefilter("ignore", category=RuntimeWarning)
|
|
348
|
+
tmp = np.where(np.isfinite(np.nanmean(self.data, axis=(1, 2, 3, 4))))[0]
|
|
349
|
+
start = tmp[0]
|
|
350
|
+
end = tmp[-1]+1
|
|
351
|
+
|
|
352
|
+
self.data = self.data[start:end, ...]
|
|
353
|
+
self.production_datetime = self.production_datetime[start:end]
|
|
354
|
+
|
|
355
|
+
def join(self, meteoRaster, strickt=False, trim=False):
|
|
356
|
+
'''
|
|
357
|
+
joins a new meteoraster to self based on production dates
|
|
358
|
+
keeps a minimum number of ensembles.
|
|
359
|
+
keeps only matching leadtimes
|
|
360
|
+
'''
|
|
361
|
+
|
|
362
|
+
if trim:
|
|
363
|
+
self.trim()
|
|
364
|
+
meteoRaster.trim()
|
|
365
|
+
|
|
366
|
+
newData = meteoRaster.data
|
|
367
|
+
newproduction_datetime = meteoRaster.production_datetime
|
|
368
|
+
try:
|
|
369
|
+
newLeadtimes = meteoRaster.leadtimes.astype(self.leadtimes.dtype)
|
|
370
|
+
except Exception as ex:
|
|
371
|
+
newLeadtimes = meteoRaster.leadtimes
|
|
372
|
+
newLatitudes = meteoRaster.latitudes
|
|
373
|
+
newLongitudes = meteoRaster.longitudes
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
# handle spatial coverage
|
|
377
|
+
if (np.abs(self.latitudes-newLatitudes)>1E-10).any() or self.data.shape[-2]!=newData.shape[-2]:
|
|
378
|
+
raise(Exception('Latitudes do not match.'))
|
|
379
|
+
|
|
380
|
+
if (np.abs(self.longitudes-newLongitudes)>1E-10).any() or self.data.shape[-1]!=newData.shape[-1]:
|
|
381
|
+
raise(Exception('Longitudes do not match.'))
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# handle ensembles
|
|
385
|
+
e0 = self.data.shape[1]
|
|
386
|
+
e1 = newData.shape[1]
|
|
387
|
+
warningMessage = 'The ensemble numbers do not match (base: %d, joint: %d). Filled missing with NaN.' % (e0, e1)
|
|
388
|
+
errorMessage = 'The ensemble numbers do not match (base: %d, joint: %d). Error issued in strickt mode.' % (e0, e1)
|
|
389
|
+
if e0>e1:
|
|
390
|
+
if strickt:
|
|
391
|
+
raise(Exception(errorMessage))
|
|
392
|
+
else:
|
|
393
|
+
warnings.warn(warningMessage)
|
|
394
|
+
shape = newData.shape
|
|
395
|
+
data = np.empty([shape[0], e0] + list(shape[2:])) * np.nan
|
|
396
|
+
data[:, :e1, :, :, :] = newData
|
|
397
|
+
newData = data
|
|
398
|
+
elif e0<e1:
|
|
399
|
+
if strickt:
|
|
400
|
+
raise(Exception(errorMessage))
|
|
401
|
+
else:
|
|
402
|
+
warnings.warn(warningMessage)
|
|
403
|
+
shape = self.data.shape
|
|
404
|
+
data = np.empty([shape[0], e1] + list(shape[2:])) * np.nan
|
|
405
|
+
data[:, :e0, :, :, :] = self.data
|
|
406
|
+
self.data = data
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
# handle leadtimes
|
|
410
|
+
v0 = np.where(np.isin(self.leadtimes, newLeadtimes))[0]
|
|
411
|
+
v1 = np.where(np.isin(newLeadtimes, self.leadtimes))[0]
|
|
412
|
+
|
|
413
|
+
if v0.size!=self.leadtimes.size or v1.size!=newLeadtimes.size:
|
|
414
|
+
if strickt:
|
|
415
|
+
raise(Exception('Leadtimes do not all match (base: %d, joint: %d). Error issued in strickt mode.' % (self.leadtimes.size, newLeadtimes.size)))
|
|
416
|
+
else:
|
|
417
|
+
warnings.warn('Leadtimes do not all match (base: %d, joint: %d). Keeping only matching values.' % (self.leadtimes.size, newLeadtimes.size))
|
|
418
|
+
self.leadtimes = self.leadtimes[v0]
|
|
419
|
+
self.data = self.data[:, :, v0, :, :]
|
|
420
|
+
newData = newData[:, :, v1, :, :]
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
# join
|
|
424
|
+
if self.leadtimes.shape != newLeadtimes.shape or not np.array_equal(self.leadtimes, newLeadtimes):
|
|
425
|
+
raise(Exception('Leadtimes do not match. Not implemented'))
|
|
426
|
+
|
|
427
|
+
if self.production_datetime[-1]<newproduction_datetime[0]:
|
|
428
|
+
# self if first
|
|
429
|
+
self.production_datetime = np.concatenate((self.production_datetime, newproduction_datetime))
|
|
430
|
+
self.data = np.concatenate((self.data, newData), axis=0)
|
|
431
|
+
|
|
432
|
+
elif self.production_datetime[0]>newproduction_datetime[-1]:
|
|
433
|
+
# joint is first
|
|
434
|
+
self.production_datetime = np.concatenate((newproduction_datetime, self.production_datetime))
|
|
435
|
+
self.data = np.concatenate((newData, self.data), axis=0)
|
|
436
|
+
else:
|
|
437
|
+
# mixed/interleaved production dates: align on the union of dates, fill NaN, new data overwrites on overlap
|
|
438
|
+
all_dates = np.sort(np.unique(np.concatenate((self.production_datetime, newproduction_datetime))))
|
|
439
|
+
target_shape = (all_dates.size,) + tuple(self.data.shape[1:])
|
|
440
|
+
target_dtype = np.result_type(self.data.dtype, np.float32)
|
|
441
|
+
merged = np.full(target_shape, np.nan, dtype=target_dtype)
|
|
442
|
+
|
|
443
|
+
# place existing data
|
|
444
|
+
idx_self = np.searchsorted(all_dates, self.production_datetime)
|
|
445
|
+
merged[idx_self, ...] = self.data.astype(target_dtype, copy=False)
|
|
446
|
+
|
|
447
|
+
# place new data (overwrite on overlap)
|
|
448
|
+
idx_new = np.searchsorted(all_dates, newproduction_datetime)
|
|
449
|
+
newData_cast = newData.astype(target_dtype, copy=False)
|
|
450
|
+
existing = merged[idx_new, ...]
|
|
451
|
+
nan_mask = np.isnan(existing)
|
|
452
|
+
if nan_mask.any():
|
|
453
|
+
existing[nan_mask] = newData_cast[nan_mask]
|
|
454
|
+
merged[idx_new, ...] = existing
|
|
455
|
+
|
|
456
|
+
self.production_datetime = all_dates
|
|
457
|
+
self.data = merged
|
|
458
|
+
|
|
459
|
+
def adjust_leadtimes(self, period='months'):
|
|
460
|
+
'''
|
|
461
|
+
Adjust leadtimes to relative time steps (useful for monthly)
|
|
462
|
+
'''
|
|
463
|
+
self._diag('Adjusting lead times...', self.verbose)
|
|
464
|
+
|
|
465
|
+
periodDefinition = {
|
|
466
|
+
'months': pd.Series([pd.DateOffset(months=i) for i in range(1, math.ceil(pd.Series(self.leadtimes[-1]).dt.days.values[0]/30)+1)]),
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if period not in periodDefinition.keys():
|
|
470
|
+
raise(Exception('%s is not a valid period.' % period))
|
|
471
|
+
|
|
472
|
+
warnings.warn('the function adjustLeadTimes() is experimental.')
|
|
473
|
+
|
|
474
|
+
times = self.production_datetime[0] + self.leadtimes
|
|
475
|
+
validTimes = pd.Timestamp(self.production_datetime[0]) + periodDefinition[period]
|
|
476
|
+
|
|
477
|
+
idxs = self.closestIdx(times, validTimes)
|
|
478
|
+
validPeriods = periodDefinition[period][:idxs.max()+1]
|
|
479
|
+
|
|
480
|
+
tmp = list(self.data.shape)
|
|
481
|
+
data = np.empty(tmp[:2] + [validPeriods.size] + tmp[-2:]) * np.nan
|
|
482
|
+
|
|
483
|
+
for i0 in idxs:
|
|
484
|
+
dataForLeadtime = self.data[:, :, idxs==i0, :, :]
|
|
485
|
+
finite = np.isfinite(dataForLeadtime)
|
|
486
|
+
count = np.sum(finite, axis=2)
|
|
487
|
+
if count.max()>1:
|
|
488
|
+
warnings.warn('Overlapping information detected when adjusting lead times (%s).' % str(validPeriods[i0]))
|
|
489
|
+
|
|
490
|
+
tmp = np.nanmean(dataForLeadtime, axis=2)
|
|
491
|
+
tmp[count==0] = np.nan
|
|
492
|
+
|
|
493
|
+
if np.isnan(tmp).all():
|
|
494
|
+
warnings.warn('Unused Lead time (%s)' % str(validPeriods[i0]))
|
|
495
|
+
|
|
496
|
+
data[:, :, i0, :, :] = tmp
|
|
497
|
+
|
|
498
|
+
self.data = data
|
|
499
|
+
if period=='months':
|
|
500
|
+
self.leadtimes = np.array([pd.DateOffset(months=i0.months-1) for i0 in validPeriods])
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
self._diag(' Done.', self.verbose)
|
|
504
|
+
|
|
505
|
+
def get_missing(self):
|
|
506
|
+
'''
|
|
507
|
+
Provides a diagnostic of missing data
|
|
508
|
+
'''
|
|
509
|
+
|
|
510
|
+
missingFraction = np.isnan(self.data).mean(axis=(-2, -1))
|
|
511
|
+
missingFraction = pd.DataFrame(missingFraction.reshape((missingFraction.shape[0], np.prod(missingFraction.shape[1:]))), index=self.production_datetime)
|
|
512
|
+
missingFraction.index.name = 'production_datetime'
|
|
513
|
+
missingFraction.columns = pd.MultiIndex.from_product((self.leadtimes, np.arange(self.data.shape[1])), names=['leadtime', 'ensemble_member'])
|
|
514
|
+
|
|
515
|
+
return missingFraction.transpose()
|
|
516
|
+
|
|
517
|
+
def get_values_from_latlon(self, lat, lon):
|
|
518
|
+
'''
|
|
519
|
+
Returns a dataframe with data from the pixel closest to the desired point
|
|
520
|
+
'''
|
|
521
|
+
|
|
522
|
+
idxLat, idxLon = self._getClosestPixels(lat, lon)
|
|
523
|
+
|
|
524
|
+
if idxLat.size>1 or idxLon.size>1:
|
|
525
|
+
warnings.warn('Several pixels are "closest". Averaging results.')
|
|
526
|
+
tmp = []
|
|
527
|
+
for iLat, iLon in zip(idxLat, idxLon):
|
|
528
|
+
tmp.append(np.expand_dims(self.data[:, :, :, iLat, iLon], axis=0))
|
|
529
|
+
tmp = np.concatenate(tmp, axis=0)
|
|
530
|
+
tmp = np.nanmean(tmp, axis=0) # [production_datetime, ensemble_member, leadtimes]
|
|
531
|
+
|
|
532
|
+
data = pd.DataFrame(tmp.reshape(tmp.shape[0]*tmp.shape[1], tmp.shape[2]))
|
|
533
|
+
data.index = pd.MultiIndex.from_product((self.production_datetime.ravel(), np.arange(self.data.shape[1])), names=['production_datetime', 'ensemble_member'])
|
|
534
|
+
data.columns = self.leadtimes
|
|
535
|
+
data.columns.name = 'leadtime'
|
|
536
|
+
|
|
537
|
+
data = data.unstack('ensemble_member')
|
|
538
|
+
|
|
539
|
+
return data
|
|
540
|
+
|
|
541
|
+
@staticmethod
|
|
542
|
+
def get_values_from_latlon_by_event(production_date_dataframe):
|
|
543
|
+
'''
|
|
544
|
+
!!!!! Change the index name to Event dates
|
|
545
|
+
'''
|
|
546
|
+
|
|
547
|
+
ensembles = production_date_dataframe.columns.get_level_values('ensemble_member').unique()
|
|
548
|
+
full_data_ = []
|
|
549
|
+
for ensemble in ensembles:
|
|
550
|
+
tmp = production_date_dataframe.loc[:, production_date_dataframe.columns.get_level_values('ensemble_member')==ensemble]
|
|
551
|
+
data_ = []
|
|
552
|
+
for i0 in range(tmp.shape[1]):
|
|
553
|
+
tmp_ = tmp.iloc[:, [i0]]
|
|
554
|
+
tmp_.index = tmp_.index + tmp_.columns.get_level_values('leadtime')[0]
|
|
555
|
+
data_.append(tmp_)
|
|
556
|
+
full_data_.append(pd.concat(data_, axis=1))
|
|
557
|
+
|
|
558
|
+
return pd.concat(full_data_, axis=1)
|
|
559
|
+
|
|
560
|
+
def plot_availability(self, missing=None, individualMembers=False):
|
|
561
|
+
'''
|
|
562
|
+
Plots the availability of the data
|
|
563
|
+
'''
|
|
564
|
+
|
|
565
|
+
if isinstance(missing, type(None)):
|
|
566
|
+
missing = self.get_missing()
|
|
567
|
+
|
|
568
|
+
available = missing < 0.95
|
|
569
|
+
|
|
570
|
+
if individualMembers:
|
|
571
|
+
tmp = available.stack().to_frame()
|
|
572
|
+
else:
|
|
573
|
+
virtualLevels = pd.to_datetime(self.production_datetime[0]) + self.leadtimes
|
|
574
|
+
available.index = available.index.set_levels(virtualLevels, level=0)
|
|
575
|
+
tmp = available.groupby(level=0).sum().transpose().stack().to_frame()
|
|
576
|
+
tmp.index = tmp.index.set_levels(self.leadtimes, level=1)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
tmp.columns = ['available']
|
|
580
|
+
tmp = tmp.loc[tmp['available']>0,:].reset_index()
|
|
581
|
+
tmp.loc[:, 'event_dates'] = tmp.loc[:, 'production_datetime'] + tmp.loc[:, 'leadtime']
|
|
582
|
+
|
|
583
|
+
sns.set_theme(style="whitegrid")
|
|
584
|
+
if individualMembers:
|
|
585
|
+
g = sns.relplot(
|
|
586
|
+
data=tmp,
|
|
587
|
+
x='event_dates', y='production_datetime', hue='ensemble_member',
|
|
588
|
+
palette='plasma',
|
|
589
|
+
)
|
|
590
|
+
else:
|
|
591
|
+
g = sns.relplot(
|
|
592
|
+
data=tmp,
|
|
593
|
+
x='event_dates', y='production_datetime', hue='available', size='available',
|
|
594
|
+
palette='viridis',
|
|
595
|
+
sizes={s: s*10 for s in np.arange(1,max((11, tmp.loc[:, 'available'].max()+1)))},
|
|
596
|
+
#===================================================================
|
|
597
|
+
# size_order=np.arange(1, max((11, tmp.loc[:, 'available'].max()+1)))
|
|
598
|
+
#===================================================================
|
|
599
|
+
)
|
|
600
|
+
g.ax.xaxis.grid(True, "minor", linewidth=.25)
|
|
601
|
+
g.ax.yaxis.grid(True, "minor", linewidth=.25)
|
|
602
|
+
|
|
603
|
+
plt.show(block=False)
|
|
604
|
+
|
|
605
|
+
def resample_timestep(self, rule, fun=np.mean):
|
|
606
|
+
'''
|
|
607
|
+
Resamples data according to pandas conventions
|
|
608
|
+
https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.resample.html
|
|
609
|
+
|
|
610
|
+
rule='MS' resamples to the start of the month
|
|
611
|
+
'''
|
|
612
|
+
|
|
613
|
+
warnings.warn('the function resample_timestep() is experimental.')
|
|
614
|
+
|
|
615
|
+
self._diag(' Resampling time step (rule: %s, %s)...' % (rule, str(fun)), self.verbose)
|
|
616
|
+
|
|
617
|
+
# prepare a template dataframe
|
|
618
|
+
tmp = pd.DataFrame(np.arange(self.production_datetime.shape[0]), index=self.production_datetime)
|
|
619
|
+
idxMax = tmp.resample(rule=rule).max().astype(int)+1
|
|
620
|
+
|
|
621
|
+
resampled = np.empty([idxMax.shape[0]] + list(self.data.shape[1:])) * np.nan
|
|
622
|
+
idx0 = 0
|
|
623
|
+
for i0, i1 in enumerate(idxMax.values.ravel()):
|
|
624
|
+
resampled[i0, :, :, :, :] = fun(self.data[idx0:i1, :, :, :, :], axis=0, keepdims=True)
|
|
625
|
+
idx0 = i1
|
|
626
|
+
|
|
627
|
+
self.data = resampled
|
|
628
|
+
self.production_datetime = idxMax.index
|
|
629
|
+
|
|
630
|
+
self._diag(' Done (%s to %s).' % (self.production_datetime[0].strftime('%Y-%m-%d %H:%M:%S', self.verbose),
|
|
631
|
+
self.production_datetime[-1].strftime('%Y-%m-%d %H:%M:%S')))
|
|
632
|
+
|
|
633
|
+
def is_complete(self, full_ensemble:bool=True, space_completeness:bool=False) -> bool:
|
|
634
|
+
'''
|
|
635
|
+
Checks if the dataset is complete.
|
|
636
|
+
|
|
637
|
+
- Always requires every production_datetime x leadtime slice to have finite data.
|
|
638
|
+
- If full_ensemble is True, all ensemble members must be finite for each slice;
|
|
639
|
+
otherwise, at least one ensemble member with finite data is enough.
|
|
640
|
+
- If space_completeness is True, all spatial points (y, x) must be finite;
|
|
641
|
+
otherwise, at least one spatial point with finite data is enough.
|
|
642
|
+
|
|
643
|
+
:param full_ensemble: Require all ensemble members to be complete.
|
|
644
|
+
:param space_completeness: Require full spatial coverage.
|
|
645
|
+
:return: True if complete, False otherwise.
|
|
646
|
+
'''
|
|
647
|
+
if self.data is None:
|
|
648
|
+
return False
|
|
649
|
+
|
|
650
|
+
finite = np.isfinite(self.data)
|
|
651
|
+
|
|
652
|
+
# Ensemble requirement
|
|
653
|
+
if full_ensemble:
|
|
654
|
+
finite = finite.all(axis=1) # -> [prod, leadtime, y, x]
|
|
655
|
+
else:
|
|
656
|
+
finite = finite.any(axis=1) # -> [prod, leadtime, y, x]
|
|
657
|
+
|
|
658
|
+
# Spatial requirement
|
|
659
|
+
if space_completeness:
|
|
660
|
+
finite = finite.all(axis=(-2, -1)) # -> [prod, leadtime]
|
|
661
|
+
else:
|
|
662
|
+
finite = finite.any(axis=(-2, -1)) # -> [prod, leadtime]
|
|
663
|
+
|
|
664
|
+
# Temporal requirement: every production_datetime and every leadtime must be complete
|
|
665
|
+
return finite.all()
|
|
666
|
+
|
|
667
|
+
def get_complete_index(self, full_ensemble:bool=True, space_completeness:bool=False) -> pd.DataFrame:
|
|
668
|
+
'''
|
|
669
|
+
|
|
670
|
+
'''
|
|
671
|
+
|
|
672
|
+
self._diag('Retrieving complete index...', self.verbose)
|
|
673
|
+
|
|
674
|
+
finite = np.isfinite(self.data)
|
|
675
|
+
|
|
676
|
+
# Ensemble requirement
|
|
677
|
+
if full_ensemble:
|
|
678
|
+
finite = finite.all(axis=1) # -> [production_datetime, leadtime, y, x]
|
|
679
|
+
else:
|
|
680
|
+
finite = finite.any(axis=1) # -> [production_datetime, leadtime, y, x]
|
|
681
|
+
|
|
682
|
+
# Spatial requirement
|
|
683
|
+
if space_completeness:
|
|
684
|
+
finite = finite.all(axis=(-2, -1)) # -> [production_datetime, leadtime] or [production_datetime, ensemble_member, leadtime]
|
|
685
|
+
else:
|
|
686
|
+
finite = finite.any(axis=(-2, -1)) # -> [production_datetime, leadtime] or [production_datetime, ensemble_member, leadtime]
|
|
687
|
+
|
|
688
|
+
completeness_index = pd.DataFrame(finite, index=self.production_datetime, columns=self.leadtimes)
|
|
689
|
+
completeness_index.index.name = 'production_datetime'
|
|
690
|
+
completeness_index.columns.name = 'leadtime'
|
|
691
|
+
|
|
692
|
+
return completeness_index
|
|
693
|
+
|
|
694
|
+
@classmethod
|
|
695
|
+
def get_completeness(cls, file) -> bool:
|
|
696
|
+
'''
|
|
697
|
+
Docstring for check_completeness
|
|
698
|
+
|
|
699
|
+
:param file: Description
|
|
700
|
+
:return: Description
|
|
701
|
+
:rtype: bool
|
|
702
|
+
'''
|
|
703
|
+
|
|
704
|
+
try:
|
|
705
|
+
with xr.open_dataset(Path(file), decode_times=False, engine=cls.ENGINE) as ds:
|
|
706
|
+
return ds.attrs['complete'].lower() in ['true', 'yes']
|
|
707
|
+
except Exception as ex:
|
|
708
|
+
raise(ex)
|
|
709
|
+
|
|
710
|
+
def save(self, file, complevel=1, complete=None) -> None:
|
|
711
|
+
'''
|
|
712
|
+
Saves a netcdf file (.nc)
|
|
713
|
+
'''
|
|
714
|
+
|
|
715
|
+
if not isinstance(file, Path):
|
|
716
|
+
file = Path(file)
|
|
717
|
+
|
|
718
|
+
self._diag('Saving %s...' % file, self.verbose)
|
|
719
|
+
|
|
720
|
+
data_array = self.to_xarray()
|
|
721
|
+
ds = xr.Dataset({self.variable: data_array})
|
|
722
|
+
ds['lat'].attrs['standard_name'] = 'latitude'
|
|
723
|
+
ds['lon'].attrs['standard_name'] = 'longitude'
|
|
724
|
+
|
|
725
|
+
ds.attrs = {k: v for k, v in self.ds_attrs.items()}
|
|
726
|
+
ds.attrs['history'] = data_array.attrs['saved']
|
|
727
|
+
if complete is None:
|
|
728
|
+
ds.attrs['complete'] = str(self.is_complete())
|
|
729
|
+
else:
|
|
730
|
+
ds.attrs['complete'] = str(complete)
|
|
731
|
+
|
|
732
|
+
tmp_file = None
|
|
733
|
+
try:
|
|
734
|
+
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
|
735
|
+
tmp_file = Path(tmp.name)
|
|
736
|
+
|
|
737
|
+
if complevel>0:
|
|
738
|
+
encoding = {
|
|
739
|
+
self.variable: {"zlib": True, "complevel": complevel},
|
|
740
|
+
}
|
|
741
|
+
ds.to_netcdf(tmp_file, encoding=encoding, engine=self.ENGINE)
|
|
742
|
+
else:
|
|
743
|
+
ds.to_netcdf(tmp_file, engine=self.ENGINE)
|
|
744
|
+
|
|
745
|
+
shutil.move(tmp_file, file)
|
|
746
|
+
except Exception:
|
|
747
|
+
if tmp_file and tmp_file.exists():
|
|
748
|
+
try:
|
|
749
|
+
tmp_file.unlink()
|
|
750
|
+
except Exception:
|
|
751
|
+
pass
|
|
752
|
+
raise
|
|
753
|
+
|
|
754
|
+
self._diag(' Saved.', self.verbose)
|
|
755
|
+
|
|
756
|
+
def to_xarray(self) -> xr.DataArray:
|
|
757
|
+
'''
|
|
758
|
+
Docstring for to_xarray
|
|
759
|
+
|
|
760
|
+
:param self: Description
|
|
761
|
+
:return: Description
|
|
762
|
+
:rtype: DataArray
|
|
763
|
+
'''
|
|
764
|
+
|
|
765
|
+
leadtimes = self.leadtimes
|
|
766
|
+
leadtime_format = 'pd.Timedelta'
|
|
767
|
+
if isinstance(leadtimes[-1], pd.DateOffset):
|
|
768
|
+
if hasattr(leadtimes[-1], 'months'):
|
|
769
|
+
leadtimes = [i.months for i in leadtimes]
|
|
770
|
+
leadtime_format = 'pd.DateOffset:months'
|
|
771
|
+
else:
|
|
772
|
+
leadtimes = [i.years for i in leadtimes]
|
|
773
|
+
leadtime_format = 'pd.DateOffset:years'
|
|
774
|
+
|
|
775
|
+
data_array = xr.DataArray(
|
|
776
|
+
data=self.data,
|
|
777
|
+
dims=['production_datetime', 'ensemble_member', 'leadtime', 'y', 'x'],
|
|
778
|
+
coords={
|
|
779
|
+
'production_datetime': self.production_datetime, # pd.Timestamp
|
|
780
|
+
'ensemble_member': np.arange(self.data.shape[1], dtype=int), # int
|
|
781
|
+
'leadtime': leadtimes, # pd.Timedelta / DateOffset
|
|
782
|
+
'lat': (['y', 'x'], self.latitudes), # float
|
|
783
|
+
'lon': (['y', 'x'], self.longitudes), # float
|
|
784
|
+
'y': np.arange(self.data.shape[-2], dtype=int),
|
|
785
|
+
'x': np.arange(self.data.shape[-1], dtype=int),
|
|
786
|
+
'leadtime_format': leadtime_format,
|
|
787
|
+
},
|
|
788
|
+
name=self.variable,
|
|
789
|
+
attrs={'units': self.units, 'variable': self.variable, 'saved': f'Saved by {self.__class__.__name__} v{self.VERSION} @ {pd.Timestamp.now().isoformat()}'},
|
|
790
|
+
)
|
|
791
|
+
|
|
792
|
+
return data_array
|
|
793
|
+
|
|
794
|
+
@classmethod
|
|
795
|
+
def load(cls, file, verbose=None) -> "MeteoRaster":
|
|
796
|
+
'''
|
|
797
|
+
Loads a MeteoRaster from a NetCDF (.nc / .nct) file written by save().
|
|
798
|
+
'''
|
|
799
|
+
|
|
800
|
+
if not isinstance(file, Path):
|
|
801
|
+
file = Path(file)
|
|
802
|
+
|
|
803
|
+
kwargs = {}
|
|
804
|
+
if verbose is None:
|
|
805
|
+
verbose = cls.VERBOSE
|
|
806
|
+
else:
|
|
807
|
+
kwargs['verbose'] = verbose
|
|
808
|
+
cls._diag('Loading %s...' % file, verbose)
|
|
809
|
+
|
|
810
|
+
if file.suffix.lower() not in {'.nc', '.nct'}:
|
|
811
|
+
raise ValueError('Only NetCDF files written by save() are supported.')
|
|
812
|
+
|
|
813
|
+
with xr.open_dataset(file, engine=cls.ENGINE) as ds:
|
|
814
|
+
# pick the first data variable if name is unknown
|
|
815
|
+
var_name = cls.__detect_variable_name(ds)
|
|
816
|
+
da = ds[var_name]
|
|
817
|
+
|
|
818
|
+
units = da.attrs.get('units', 'unknown')
|
|
819
|
+
variable = da.attrs.get('variable', 'unknown')
|
|
820
|
+
da_attrs = ds.attrs
|
|
821
|
+
ds_attrs = ds.attrs
|
|
822
|
+
|
|
823
|
+
leadtimes = ds['leadtime'].values
|
|
824
|
+
if hasattr(ds['leadtime'], 'leadtime_format'):
|
|
825
|
+
leadtime_format = str(ds['leadtime'].leadtime_format.values)
|
|
826
|
+
if leadtime_format.split(':')[0]=='pd.DateOffset':
|
|
827
|
+
leadtimes = pd.Index([pd.DateOffset(**{leadtime_format.split(':')[1]: lt}) for lt in leadtimes])
|
|
828
|
+
|
|
829
|
+
meteo = cls(
|
|
830
|
+
data=da.values,
|
|
831
|
+
latitudes=ds['lat'].values,
|
|
832
|
+
longitudes=ds['lon'].values,
|
|
833
|
+
production_datetime=ds['production_datetime'].values,
|
|
834
|
+
leadtimes=leadtimes,
|
|
835
|
+
units=units,
|
|
836
|
+
variable=variable,
|
|
837
|
+
da_attrs=da_attrs,
|
|
838
|
+
ds_attrs=ds_attrs,
|
|
839
|
+
**kwargs,
|
|
840
|
+
)
|
|
841
|
+
|
|
842
|
+
cls._diag(' Loaded.', verbose)
|
|
843
|
+
|
|
844
|
+
return meteo
|
|
845
|
+
|
|
846
|
+
@staticmethod
|
|
847
|
+
def __detect_variable_name(ds: xr.Dataset) -> str:
|
|
848
|
+
# Require single-variable datasets written by save()
|
|
849
|
+
if len(ds.data_vars) != 1:
|
|
850
|
+
raise ValueError('NetCDF file must contain exactly one data variable written by save().')
|
|
851
|
+
return next(iter(ds.data_vars))
|
|
852
|
+
|
|
853
|
+
def copy(self):
|
|
854
|
+
'''
|
|
855
|
+
Makes a copy of a MeteoRaster file
|
|
856
|
+
'''
|
|
857
|
+
|
|
858
|
+
mr = MeteoRaster(data=self.data.copy(),
|
|
859
|
+
latitudes=self.latitudes.copy(), longitudes=self.longitudes.copy(),
|
|
860
|
+
production_datetime=self.production_datetime.copy(), leadtimes=self.leadtimes.copy(),
|
|
861
|
+
da_attrs=self.da_attrs, ds_attrs=self.ds_attrs,
|
|
862
|
+
)
|
|
863
|
+
return mr
|
|
864
|
+
|
|
865
|
+
def _getClosestPixels(self, lat, lon):
|
|
866
|
+
'''
|
|
867
|
+
Returns the indexes of the pixel closer to a point
|
|
868
|
+
'''
|
|
869
|
+
|
|
870
|
+
dist = ((self.longitudes-lon)**2+(self.latitudes-lat)**2)**0.5
|
|
871
|
+
idx_lat, idx_lon = np.where(dist==dist.ravel().min())
|
|
872
|
+
|
|
873
|
+
return (idx_lat, idx_lon)
|
|
874
|
+
|
|
875
|
+
@staticmethod
|
|
876
|
+
def closestIdx(X, Y):
|
|
877
|
+
'''
|
|
878
|
+
Returns the list of Y indexes closest the values of X
|
|
879
|
+
Dimension of X.
|
|
880
|
+
Indexes to be used in Y.
|
|
881
|
+
'''
|
|
882
|
+
|
|
883
|
+
idxs = np.empty_like(X, dtype=int)
|
|
884
|
+
for i0, x0 in enumerate(X):
|
|
885
|
+
idxs[i0] = np.argmin(np.abs(x0-np.array(Y)))
|
|
886
|
+
|
|
887
|
+
return idxs
|
|
888
|
+
|
|
889
|
+
def _flipLatitude(self):
|
|
890
|
+
'''
|
|
891
|
+
flips the latitude of the raster
|
|
892
|
+
'''
|
|
893
|
+
|
|
894
|
+
self.data = np.flip(self.data, axis=3)
|
|
895
|
+
self.latitudes = np.flip(self.latitudes, axis=0)
|
|
896
|
+
self.longitudes = np.flip(self.longitudes, axis=0)
|
|
897
|
+
|
|
898
|
+
self._diag('Flipped latitudes.', self.verbose)
|
|
899
|
+
|
|
900
|
+
def _fixStartAtGreenwich(self):
|
|
901
|
+
'''
|
|
902
|
+
modifies rasters with longitudes starting at Greenwich to the more conveninent -180E.
|
|
903
|
+
'''
|
|
904
|
+
|
|
905
|
+
westIndexes = np.arange(len(self.longitudes))[self.longitudes>180]
|
|
906
|
+
eastIndexes = np.arange(len(self.longitudes))[self.longitudes<=180]
|
|
907
|
+
|
|
908
|
+
longitudes = np.concatenate((self.longitudes[westIndexes]-360, self.longitudes[eastIndexes]))
|
|
909
|
+
|
|
910
|
+
westData = self.data[:, :, : ,: ,westIndexes]
|
|
911
|
+
eastData = self.data[:, :, : ,: ,eastIndexes]
|
|
912
|
+
|
|
913
|
+
data = np.concatenate((westData, eastData), axis=-1)
|
|
914
|
+
|
|
915
|
+
self.longitudes = longitudes
|
|
916
|
+
self.data = data
|
|
917
|
+
|
|
918
|
+
self._diag('Fixed start at Greenwich.', self.verbose)
|
|
919
|
+
|
|
920
|
+
def __groupByMatrix(self, coverage, columns, elementwise=False):
|
|
921
|
+
'''
|
|
922
|
+
Groups the data by a coverage matrix (see groupByKML for matrix definition)
|
|
923
|
+
'''
|
|
924
|
+
|
|
925
|
+
self._diag('Executing spatial aggregation...', self.verbose)
|
|
926
|
+
|
|
927
|
+
#Verification of valid tiles (assumes first coverage zone to be representative (for speed and memory)
|
|
928
|
+
val_ = np.isfinite(self.data).astype(bool)
|
|
929
|
+
cvr = np.tile(coverage[:, :, :, :, :, 0].astype(np.single), list(val_.shape[:(val_.ndim-2)]) + [1, 1])
|
|
930
|
+
val = np.nansum(cvr*val_, axis=(-2, -1))>0.8
|
|
931
|
+
val = np.tile(np.expand_dims(val,-1), [1]*val.ndim + [coverage.shape[-1]])
|
|
932
|
+
|
|
933
|
+
if elementwise:
|
|
934
|
+
agg = np.empty(list(self.data.shape[:3]) + [len(columns)]).astype(np.single) * np.nan
|
|
935
|
+
|
|
936
|
+
data_single = self.data.astype(np.single)
|
|
937
|
+
coverage_single = coverage.astype(np.single)
|
|
938
|
+
|
|
939
|
+
for i0 in range(len(columns)):
|
|
940
|
+
agg[:, :, :, i0] = np.nansum(data_single * np.tile(coverage_single[:, :, :, :, :, i0], list(data_single.shape[:(self.data.ndim-2)]) + [1, 1]), axis=(-2, -1))
|
|
941
|
+
self._diag(' % 5.1f%%' % ((i0+1)/len(columns)*100), self.verbose)
|
|
942
|
+
else:
|
|
943
|
+
#Creation of matrices for calculation
|
|
944
|
+
dat = np.tile(np.expand_dims(self.data.astype(np.single), axis=-1), [1]*self.data.ndim + [coverage.shape[-1]])
|
|
945
|
+
cvr = np.tile(coverage.astype(np.single), list(dat.shape[:(dat.ndim-3)]) + [1, 1, 1])
|
|
946
|
+
|
|
947
|
+
#Correction of the coveragefor missing data
|
|
948
|
+
dat_finite = np.isfinite(dat).astype(bool)
|
|
949
|
+
cvr_finite = cvr.copy()
|
|
950
|
+
cvr_finite[~dat_finite] = 0
|
|
951
|
+
weight_update = 1 / cvr_finite.sum(axis=(-3, -2))
|
|
952
|
+
weight_update = np.tile(weight_update[:, :, :, np.newaxis, np.newaxis, :], [1, 1, 1] + list(cvr.shape[-3:-1]) + [1])
|
|
953
|
+
|
|
954
|
+
dat *= cvr
|
|
955
|
+
dat *= weight_update
|
|
956
|
+
|
|
957
|
+
cvr = None
|
|
958
|
+
gc.collect()
|
|
959
|
+
|
|
960
|
+
#Calculation with garbage collection
|
|
961
|
+
agg = np.nansum(dat, axis=(-3, -2))
|
|
962
|
+
dat = None
|
|
963
|
+
gc.collect()
|
|
964
|
+
|
|
965
|
+
#Mark invalid tiles as NaN
|
|
966
|
+
agg[~val] = np.nan
|
|
967
|
+
|
|
968
|
+
tmp = np.reshape(agg, (agg.shape[0], np.prod(agg.shape[1:])), order='F')
|
|
969
|
+
agg = pd.DataFrame(tmp, index=self.production_datetime.ravel())
|
|
970
|
+
agg.columns = columns
|
|
971
|
+
agg.index.name = 'Production date'
|
|
972
|
+
|
|
973
|
+
self._diag(' Done.', self.verbose)
|
|
974
|
+
|
|
975
|
+
return agg
|
|
976
|
+
|
|
977
|
+
def __groupByQuantile(self, coverage, columns, resampling, quantiles, precision=0.1):
|
|
978
|
+
'''
|
|
979
|
+
Groups the data by quantiles in time and space
|
|
980
|
+
|
|
981
|
+
precision should be between 0 and 1.
|
|
982
|
+
computation time will be proportional to 1/precision
|
|
983
|
+
'''
|
|
984
|
+
|
|
985
|
+
self._diag('Executing spatial aggregation...', self.verbose)
|
|
986
|
+
|
|
987
|
+
if not resampling is None:
|
|
988
|
+
resampled_idx = pd.DataFrame(self.production_datetime, self.production_datetime).resample(resampling).indices
|
|
989
|
+
time_idx = list(resampled_idx.keys())
|
|
990
|
+
time_groups = [i for _, i in resampled_idx.items()]
|
|
991
|
+
else:
|
|
992
|
+
time_idx = self.production_datetime
|
|
993
|
+
time_groups = [[i] for i, _ in enumerate(time_idx)]
|
|
994
|
+
|
|
995
|
+
#Creation of matrices for calculation
|
|
996
|
+
dat = np.tile(np.expand_dims(self.data.astype(np.single), axis=-1), [1]*self.data.ndim + [coverage.shape[-1]])
|
|
997
|
+
coverage_ = coverage.astype(np.single)
|
|
998
|
+
coverage_[coverage_==0] = np.nan
|
|
999
|
+
gc.collect()
|
|
1000
|
+
|
|
1001
|
+
#Calculation with garbage collection
|
|
1002
|
+
agg_ = []
|
|
1003
|
+
for g0 in time_groups:
|
|
1004
|
+
data = dat[g0,:, :, :, :].copy()
|
|
1005
|
+
|
|
1006
|
+
# Verification of valid tiles (assumes first coverage zone to be representative (for speed and memory)
|
|
1007
|
+
cvr = np.tile(coverage_, list(data.shape[:(dat.ndim-3)]) + [1, 1, 1])
|
|
1008
|
+
val = cvr.copy()
|
|
1009
|
+
val[val<=0] = np.nan
|
|
1010
|
+
val[val>0] = 1
|
|
1011
|
+
|
|
1012
|
+
data_ = data * val
|
|
1013
|
+
|
|
1014
|
+
folds = np.round(cvr/precision)
|
|
1015
|
+
n_folds = int(np.nanmax(folds))
|
|
1016
|
+
data_unfolded = np.tile(np.expand_dims(data, 0), [n_folds] + [1] * data.ndim) * np.nan
|
|
1017
|
+
tmp = data_.copy()
|
|
1018
|
+
for i1 in range(n_folds):
|
|
1019
|
+
tmp[folds<i1+1] = np.nan
|
|
1020
|
+
data_unfolded[i1,:,:,:,:,:,:] = tmp
|
|
1021
|
+
|
|
1022
|
+
# Check what pixels are valid (more than 80% values available)
|
|
1023
|
+
val = np.nansum(np.isfinite(data_), axis=(0, -3, -2)) / np.nansum(np.isfinite(val), axis=(0, -3, -2)) > 0.8
|
|
1024
|
+
val = np.tile(np.expand_dims(val, (0, -1)), [1]*(val.ndim + 1) + [len(quantiles)])
|
|
1025
|
+
|
|
1026
|
+
# Calculate the quantiles
|
|
1027
|
+
raise(Exception('This must be reviewed.'))
|
|
1028
|
+
tmp = np.rollaxis(np.nanquantile(np.expand_dims(data_unfolded, 0), quantiles, axis=(1, 2, -3, -2)), 0, len(quantiles)) #[time, lead, ensemble, zones, quantiles]
|
|
1029
|
+
tmp[~val] = np.nan
|
|
1030
|
+
agg_.append(tmp)
|
|
1031
|
+
|
|
1032
|
+
agg__ = np.r_[agg_]
|
|
1033
|
+
|
|
1034
|
+
tmp = np.reshape(agg__, (agg__.shape[0], np.prod(agg__.shape[1:])), order='F')
|
|
1035
|
+
agg = pd.DataFrame(tmp, index=time_idx)
|
|
1036
|
+
|
|
1037
|
+
tmp = pd.concat([columns.to_frame()]*len(quantiles))
|
|
1038
|
+
tmp.loc[:, 'Quantile'] = np.array([[q]*np.prod(agg__.shape[1:-1]) for q in quantiles]).ravel()
|
|
1039
|
+
agg.columns = pd.MultiIndex.from_frame(tmp)
|
|
1040
|
+
agg.index.name = 'Production date'
|
|
1041
|
+
agg.sort_index(axis=1, inplace=True)
|
|
1042
|
+
|
|
1043
|
+
#=======================================================================
|
|
1044
|
+
# agg.to_clipboard(excel=True)
|
|
1045
|
+
#=======================================================================
|
|
1046
|
+
|
|
1047
|
+
self._diag(' Done.', self.verbose)
|
|
1048
|
+
|
|
1049
|
+
return agg
|
|
1050
|
+
|
|
1051
|
+
def __coverageMatrixFromKML(self, kml, nameField=None, buffer=2, normalize_by_area=True):
|
|
1052
|
+
'''
|
|
1053
|
+
Creates a coverage matrix from polygons contained in a kml file
|
|
1054
|
+
'''
|
|
1055
|
+
|
|
1056
|
+
self._diag('Creating a coverage matrix for spatial aggregation...', self.verbose)
|
|
1057
|
+
|
|
1058
|
+
# Correct kml file if required
|
|
1059
|
+
self.__correctKMLHeader(kml)
|
|
1060
|
+
|
|
1061
|
+
# Read geometry reference KML file
|
|
1062
|
+
ids = []
|
|
1063
|
+
coordinates = []
|
|
1064
|
+
document = et.parse(kml).getroot().find('Document')
|
|
1065
|
+
for i0, placemark in enumerate(document.findall('.//Placemark')):
|
|
1066
|
+
#https://docs.python.org/3/library/xml.etree.elementtree.html
|
|
1067
|
+
try:
|
|
1068
|
+
if not isinstance(nameField, type(None)):
|
|
1069
|
+
id0 = placemark.find(".//SimpleData[@name='%s']" % nameField).text
|
|
1070
|
+
else:
|
|
1071
|
+
id0 = str(i0)
|
|
1072
|
+
outer = [pd.Series(c.text.replace(' \n', '').replace('\n', '').replace('\t', '').split(' ')).str.split(',', n=2, expand=True) for c in placemark.findall('.//Polygon/outerBoundaryIs//coordinates')]
|
|
1073
|
+
outer = [c.astype(float) for c in outer]
|
|
1074
|
+
inner = [pd.Series(c.text.replace(' \n', '').replace('\n', '').replace('\t', '').split(' ')).str.split(',', n=2, expand=True) for c in placemark.findall('.//Polygon/innerBoundaryIs//coordinates')]
|
|
1075
|
+
inner = [c.astype(float) for c in inner]
|
|
1076
|
+
ids.append(id0)
|
|
1077
|
+
coordinates.append({'outer': outer, 'inner': inner})
|
|
1078
|
+
except Exception as ex:
|
|
1079
|
+
print(str(ex))
|
|
1080
|
+
|
|
1081
|
+
cvr = np.zeros([1, 1, 1] + list(self.data.shape[3:]) + [len(ids)], dtype=np.single) # the last coordinate of the cvr represents the polygon
|
|
1082
|
+
|
|
1083
|
+
longitudes = self.longitudes
|
|
1084
|
+
latitudes = self.latitudes
|
|
1085
|
+
idx_lat_ = np.tile(np.expand_dims(np.arange(longitudes.shape[0]),axis=1),(1, longitudes.shape[1])).ravel()
|
|
1086
|
+
idx_lon_ = np.tile(np.arange(longitudes.shape[1]),(longitudes.shape[0], 1)).ravel()
|
|
1087
|
+
lonInterpolator = interp.LinearNDInterpolator(np.array([idx_lat_, idx_lon_]).T, longitudes.ravel()) # f(lat_id, lon_id)
|
|
1088
|
+
latInterpolator = interp.LinearNDInterpolator(np.array([idx_lat_, idx_lon_]).T, latitudes.ravel()) # f(lat_id, lon_id)
|
|
1089
|
+
|
|
1090
|
+
# Create GDAL geometry
|
|
1091
|
+
centroids = {'x':[], 'y':[]}
|
|
1092
|
+
for i0, coords in enumerate(coordinates):
|
|
1093
|
+
try:
|
|
1094
|
+
# Add all "positive" rings
|
|
1095
|
+
poly = ogr.Geometry(ogr.wkbPolygon)
|
|
1096
|
+
for o1 in coords['outer']:
|
|
1097
|
+
for r0 in o1:
|
|
1098
|
+
polyTmp = ogr.Geometry(ogr.wkbPolygon)
|
|
1099
|
+
ring = ogr.Geometry(ogr.wkbLinearRing)
|
|
1100
|
+
for row in o1.iterrows():
|
|
1101
|
+
ring.AddPoint(row[1][0], row[1][1])
|
|
1102
|
+
_ = polyTmp.AddGeometry(ring)
|
|
1103
|
+
poly = poly.Union(polyTmp)
|
|
1104
|
+
|
|
1105
|
+
# Join all "negative rings
|
|
1106
|
+
polyHoles = ogr.Geometry(ogr.wkbPolygon)
|
|
1107
|
+
for o1 in coords['inner']:
|
|
1108
|
+
for r0 in o1:
|
|
1109
|
+
polyTmp = ogr.Geometry(ogr.wkbPolygon)
|
|
1110
|
+
ring = ogr.Geometry(ogr.wkbLinearRing)
|
|
1111
|
+
for row in o1.iterrows():
|
|
1112
|
+
ring.AddPoint(row[1][0], row[1][1])
|
|
1113
|
+
_ = polyTmp.AddGeometry(ring)
|
|
1114
|
+
polyHoles = polyHoles.Union(polyTmp)
|
|
1115
|
+
|
|
1116
|
+
# Obtain the final polygon
|
|
1117
|
+
poly = poly.SymmetricDifference(polyHoles)
|
|
1118
|
+
|
|
1119
|
+
# Select pixel subset
|
|
1120
|
+
#===============================================================
|
|
1121
|
+
# plt.figure(); plt.imshow(longitudes); plt.colorbar()
|
|
1122
|
+
# plt.figure(); plt.imshow(latitudes); plt.colorbar()
|
|
1123
|
+
# plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=self.data.squeeze().mean(axis=0).ravel()); plt.colorbar()
|
|
1124
|
+
# plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=longitudes.ravel()); plt.colorbar()
|
|
1125
|
+
# plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=latitudes.ravel()); plt.colorbar()
|
|
1126
|
+
#===============================================================
|
|
1127
|
+
|
|
1128
|
+
extent = poly.Buffer(buffer).GetEnvelope() #wesn
|
|
1129
|
+
inside = np.zeros_like(latitudes).astype(bool)
|
|
1130
|
+
inside[(longitudes>=extent[0]) & (longitudes<=extent[1]) & (latitudes>=extent[2]) & (latitudes<=extent[3])] = True
|
|
1131
|
+
tmp = np.where(inside)
|
|
1132
|
+
lat_idxs = np.arange(tmp[0].min()-1, tmp[0].max()+1)
|
|
1133
|
+
lat_idxs = lat_idxs[(lat_idxs>0) & (lat_idxs<latitudes.shape[0]-1)]
|
|
1134
|
+
lon_idxs = np.arange(tmp[1].min()-1, tmp[1].max()+1)
|
|
1135
|
+
lon_idxs = lon_idxs[(lon_idxs>0) & (lon_idxs<longitudes.shape[1]-1)]
|
|
1136
|
+
|
|
1137
|
+
#===============================================================
|
|
1138
|
+
# plt.figure(); plt.imshow(inside); plt.colorbar()
|
|
1139
|
+
# plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=inside.ravel()); plt.colorbar()
|
|
1140
|
+
#===============================================================
|
|
1141
|
+
|
|
1142
|
+
#===============================================================
|
|
1143
|
+
# extent = [extent[0]-hdLon*2, extent[1]+hdLon*2, extent[2]-hdLat*2, extent[3]+hdLat*2]
|
|
1144
|
+
# lon_idxs = np.arange(self.longitudes.shape[0])[(self.longitudes>=extent[0]) & (self.longitudes<=extent[1])]
|
|
1145
|
+
# lat_idxs = np.arange(self.latitudes.shape[0])[(self.latitudes>=extent[2]) & (self.latitudes<=extent[3])]
|
|
1146
|
+
#===============================================================
|
|
1147
|
+
|
|
1148
|
+
x, y, _ = poly.Centroid().GetPoint()
|
|
1149
|
+
centroids['x'].append(x)
|
|
1150
|
+
centroids['y'].append(y)
|
|
1151
|
+
|
|
1152
|
+
for i2 in lon_idxs:
|
|
1153
|
+
for i3 in lat_idxs:
|
|
1154
|
+
#=======================================================
|
|
1155
|
+
# lon = self.longitudes[i3, i2]
|
|
1156
|
+
# lat = self.latitudes[i3, i2]
|
|
1157
|
+
# plt.plot(lon, lat, 'rx')
|
|
1158
|
+
# plt.plot(lonInterpolator(i3-0.5,i2+0.5), latInterpolator(i3-0.5,i2+0.5), '.m')
|
|
1159
|
+
#=======================================================
|
|
1160
|
+
|
|
1161
|
+
points = []
|
|
1162
|
+
points.append((float(lonInterpolator(i3-0.5, i2+0.5)), float(latInterpolator(i3-0.5, i2+0.5))))
|
|
1163
|
+
points.append((float(lonInterpolator(i3+0.5, i2+0.5)), float(latInterpolator(i3+0.5, i2+0.5))))
|
|
1164
|
+
points.append((float(lonInterpolator(i3+0.5, i2-0.5)), float(latInterpolator(i3+0.5, i2-0.5))))
|
|
1165
|
+
points.append((float(lonInterpolator(i3-0.5, i2-0.5)), float(latInterpolator(i3-0.5, i2-0.5))))
|
|
1166
|
+
points.append((float(lonInterpolator(i3-0.5, i2+0.5)), float(latInterpolator(i3-0.5, i2+0.5))))
|
|
1167
|
+
|
|
1168
|
+
ring = ogr.Geometry(ogr.wkbLinearRing)
|
|
1169
|
+
for p in points:
|
|
1170
|
+
ring.AddPoint(*p)
|
|
1171
|
+
#===================================================
|
|
1172
|
+
# plt.plot(*p,'.b')
|
|
1173
|
+
#===================================================
|
|
1174
|
+
|
|
1175
|
+
pixel = ogr.Geometry(ogr.wkbPolygon)
|
|
1176
|
+
_ = pixel.AddGeometry(ring)
|
|
1177
|
+
|
|
1178
|
+
intersection = pixel.Intersection(poly)
|
|
1179
|
+
area = intersection.Area()
|
|
1180
|
+
|
|
1181
|
+
if area>0:
|
|
1182
|
+
cvr[0, 0, 0, i3, i2, i0] = area
|
|
1183
|
+
except Exception as ex:
|
|
1184
|
+
raise(ex)
|
|
1185
|
+
|
|
1186
|
+
# Define columns (drops redundant levels (except zone))
|
|
1187
|
+
columns = pd.MultiIndex.from_product((ids, self.leadtimes, 1+np.arange(self.data.shape[self.ENSEMBLEMEMBERpOSITION])), names=('zone', 'leadtime', 'ensemble_member'))
|
|
1188
|
+
drop = []
|
|
1189
|
+
for i0, (n0, l0) in enumerate(zip(columns.names, list(columns.levels))):
|
|
1190
|
+
if l0.shape[0]==1 and n0!='Zone':
|
|
1191
|
+
self._diag(' Level %s is redundant and will be dropped.' % n0, self.verbose)
|
|
1192
|
+
drop.append(i0)
|
|
1193
|
+
for i0 in drop[::-1]:
|
|
1194
|
+
columns = columns.droplevel(i0)
|
|
1195
|
+
|
|
1196
|
+
# Normalize
|
|
1197
|
+
if normalize_by_area:
|
|
1198
|
+
areas = cvr.sum(axis=(0, 1, 2, 3, 4))
|
|
1199
|
+
cvr /= np.tile(areas, list(cvr.shape[:-1]) + [1])
|
|
1200
|
+
else:
|
|
1201
|
+
max_pixel_area = cvr.max(axis=tuple(np.arange(cvr.ndim-1)))
|
|
1202
|
+
cvr /= np.tile(max_pixel_area, list(cvr.shape[:-1]) + [1])
|
|
1203
|
+
|
|
1204
|
+
# Prepare centroids for export
|
|
1205
|
+
centroids = pd.DataFrame(centroids, index=ids)
|
|
1206
|
+
|
|
1207
|
+
self._diag(' Done.', self.verbose)
|
|
1208
|
+
|
|
1209
|
+
return cvr, columns, centroids
|
|
1210
|
+
|
|
1211
|
+
def __correctKMLHeader(self, kml, encoding='utf-8'):
|
|
1212
|
+
'''
|
|
1213
|
+
Corrects the header of a KML file to that it can be read for model data preparation
|
|
1214
|
+
'''
|
|
1215
|
+
|
|
1216
|
+
modificationRequired = False
|
|
1217
|
+
line = ''
|
|
1218
|
+
with open(kml, 'r', encoding=encoding) as file:
|
|
1219
|
+
for line in file:
|
|
1220
|
+
if line.strip().startswith('<kml'):
|
|
1221
|
+
if line.strip() != '<kml>':
|
|
1222
|
+
modificationRequired = True
|
|
1223
|
+
break
|
|
1224
|
+
|
|
1225
|
+
if modificationRequired:
|
|
1226
|
+
if not isinstance(kml, str):
|
|
1227
|
+
kml = str(kml)
|
|
1228
|
+
|
|
1229
|
+
tmpKML = kml.replace('.kml', '__tmp.kml')
|
|
1230
|
+
with open(kml, 'r', encoding=encoding) as file_in:
|
|
1231
|
+
with open(tmpKML, 'w', encoding=encoding) as file_out:
|
|
1232
|
+
for line in file_in:
|
|
1233
|
+
if line.strip().startswith('<kml'):
|
|
1234
|
+
if line.strip() != '<kml>':
|
|
1235
|
+
line = '<kml>\n'
|
|
1236
|
+
__ = file_out.write(line)
|
|
1237
|
+
os.remove(kml)
|
|
1238
|
+
os.rename(tmpKML, kml)
|
|
1239
|
+
|
|
1240
|
+
@staticmethod
|
|
1241
|
+
def _diag(message, show=True):
|
|
1242
|
+
if show:
|
|
1243
|
+
print(message)
|
|
1244
|
+
|
|
1245
|
+
def resampleTimeStep(self, *args, **kwargs):
|
|
1246
|
+
warnings.warn('resampleTimeStep() is deprecated and will be removed in a future release; call resample_timestep() directly instead.', DeprecationWarning, stacklevel=2)
|
|
1247
|
+
self.resample_timestep(*args, **kwargs)
|
|
1248
|
+
|
|
1249
|
+
def getQuantilesFromKML(self, *args, **kwargs):
|
|
1250
|
+
warnings.warn('getQuantilesFromKML() is deprecated and will be removed in a future release; call get_quantiles_from_KML() directly instead.', DeprecationWarning, stacklevel=2)
|
|
1251
|
+
return self.get_quantiles_from_KML(*args, **kwargs)
|
|
1252
|
+
|
|
1253
|
+
def getDataFromLatLon(self, *args, **kwargs):
|
|
1254
|
+
warnings.warn('getDataFromLatLon() is deprecated and will be removed in a future release; call get_values_from_latlon() directly instead.', DeprecationWarning, stacklevel=2)
|
|
1255
|
+
return self.get_values_from_latlon(*args, **kwargs)
|
|
1256
|
+
|
|
1257
|
+
def getCropped(self, *args, **kwargs):
|
|
1258
|
+
warnings.warn('getCropped() is deprecated and will be removed in a future release; call get_cropped() directly instead.', DeprecationWarning, stacklevel=2)
|
|
1259
|
+
return self.get_cropped(*args, **kwargs)
|
|
1260
|
+
|
|
1261
|
+
def getValuesFromKML(self, *args, **kwargs):
|
|
1262
|
+
warnings.warn('getValuesFromKML() is deprecated and will be removed in a future release; call get_values_from_KML() directly instead.', DeprecationWarning, stacklevel=2)
|
|
1263
|
+
return self.get_values_from_KML(*args, **kwargs)
|
|
1264
|
+
|
|
1265
|
+
if __name__=='__main__':
|
|
1266
|
+
|
|
1267
|
+
pass
|