meteoraster 2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1267 @@
1
+ import math
2
+ import copy
3
+ import gc
4
+ import os
5
+ import warnings
6
+ import shutil
7
+ import tempfile
8
+ import numpy as np
9
+ import xarray as xr
10
+ import datetime as dt
11
+ import pandas as pd
12
+ import seaborn as sns
13
+ import matplotlib.pyplot as plt
14
+ import xml.etree.ElementTree as et
15
+ from pathlib import Path
16
+ from osgeo import ogr
17
+ import scipy.interpolate as interp
18
+ import cartopy.crs as ccrs
19
+ import cartopy.feature as cf
20
+ import cartopy.io.shapereader as shpreader
21
+
22
+ class MeteoRaster(object):
23
+ '''
24
+ Custom class to handle meteorological raster files, including ensembles and forecasts
25
+
26
+ v2.5:
27
+ Published on PyPI - installs with `pip install meteoraster`. Releases are
28
+ built and uploaded automatically on tag push (GitHub Actions + PyPI
29
+ Trusted Publishing), and attached to the GitHub release.
30
+ Licensing made consistent: the LICENSE file carried GPL-3 text while the
31
+ package metadata declared Apache-2.0. Apache-2.0 is now the license
32
+ everywhere, declared as a PEP 639 SPDX expression, with a NOTICE file.
33
+
34
+ v2.4:
35
+ Packaging modernized (pyproject.toml, optional dependencies, h5netcdf and
36
+ pytest declared as dependencies).
37
+ Cleaned up deprecated numpy/pandas idioms (np.NaN -> np.nan,
38
+ np.product -> np.prod, removed redundant axis= arguments) for
39
+ forward-compatibility with numpy 2 and pandas 3.
40
+
41
+ v2.3:
42
+ Fixed a bug in the groupByMatrix function where the coverage was not updated to reflect missing data, which could lead to incorrect aggregations. The coverage is now set to 0 for pixels where data is not valid before performing the aggregation.
43
+
44
+ v2.2:
45
+ Specifies the saving engine as 'h5netcdf'
46
+
47
+ v2.1:
48
+ Updated to VSCode
49
+ Updated to drop pickle and use cross-platfrom alternatives instead
50
+
51
+ Added testing
52
+ Removed support for reading multiple data formats
53
+ A "latest" version is compiled and published
54
+
55
+ Untested support for pd.DateOffset
56
+ '''
57
+
58
+ VERSION = '2.5'
59
+ VERBOSE = True
60
+ ENSEMBLEMEMBERpOSITION = 1
61
+
62
+ ENGINE = 'h5netcdf'
63
+
64
+ def __init__(self, data,
65
+ latitudes=None, longitudes=None, production_datetime=None, leadtimes=None,
66
+ units='unknown', variable='unknown', da_attrs={}, ds_attrs={},
67
+ verbose=None,
68
+ ):
69
+ '''
70
+ data: 5-D numpy array [production_datetime, ensemble_member, leadtime, latitude, longitude]
71
+ production dates: 1-D numpy array
72
+ ensemble members: 1-D numpy array
73
+ leadtimes: 1-D numpy array
74
+ latitudes: 1-D numpy array (center)
75
+ longitudes: 1-D numpy array (center)
76
+
77
+ alternatively, data can be a dict with the previous fields
78
+ '''
79
+
80
+ if verbose is None:
81
+ self.verbose = self.VERBOSE
82
+ else:
83
+ self.verbose = verbose
84
+
85
+ self.units = units
86
+ self.variable = variable
87
+ self.da_attrs = da_attrs
88
+ self.ds_attrs = ds_attrs
89
+ self._update_attrs()
90
+
91
+ if isinstance(data, dict):
92
+ self.data = data['data']
93
+ self.latitudes = data['latitudes']
94
+ self.longitudes = data['longitudes']
95
+ self.production_datetime = data['production_datetime']
96
+ self.leadtimes = data['leadtimes']
97
+ else:
98
+ self.data = data
99
+
100
+ if not latitudes is None:
101
+ self.latitudes = latitudes
102
+ if not longitudes is None:
103
+ self.longitudes = longitudes
104
+ if not production_datetime is None:
105
+ self.production_datetime = production_datetime
106
+ if not leadtimes is None:
107
+ self.leadtimes = leadtimes
108
+
109
+ # Fix Geomery
110
+ #Greenwish
111
+ if self.longitudes.ndim==1:
112
+ if max(self.longitudes)>180:
113
+ self._fixStartAtGreenwich()
114
+
115
+ # Process geometry so that lats and lons are stored in 2D
116
+ if self.longitudes.ndim==1 and self.latitudes.ndim==1:
117
+ self.longitudes, self.latitudes = np.meshgrid(self.longitudes, self.latitudes)
118
+
119
+ # Flip latitudes (if required)
120
+ if self.latitudes[0, 0]<self.latitudes[1, 0]:
121
+ self._flipLatitude()
122
+
123
+ def _update_attrs(self):
124
+ '''
125
+ Docstring for _update_attrs
126
+
127
+ :param self: Description
128
+ '''
129
+
130
+ if self.units == 'unknown':
131
+ if 'units' in self.da_attrs.keys():
132
+ self.units = self.da_attrs['units']
133
+ else:
134
+ self.da_attrs['units'] = self.units
135
+
136
+ if self.variable == 'unknown':
137
+ if 'variable' in self.da_attrs.keys():
138
+ self.variable = self.da_attrs['variable']
139
+ else:
140
+ self.da_attrs['variable'] = self.variable
141
+
142
+ def create_plot(self, central_longitude, projection_kwargs={}, figsize=None, projection_fun=ccrs.PlateCarree, crs=ccrs.PlateCarree(), extent=None):
143
+ '''
144
+ extent [west, east, south, north]
145
+ '''
146
+
147
+ plt.figure(figsize=figsize)
148
+ projection = projection_fun(central_longitude=central_longitude, **projection_kwargs)
149
+ ax = plt.axes(projection=projection)
150
+ if not isinstance(extent, type(None)):
151
+ ax.set_extent(extent, crs=crs)
152
+ return ax
153
+
154
+ def add_shapefile(self, ax, shapefile_path, crs=ccrs.PlateCarree(), **kwargs):
155
+ '''
156
+
157
+ '''
158
+
159
+ ax.add_geometries(shpreader.Reader(shapefile_path).geometries(), crs=crs, **kwargs)
160
+
161
+ def add_shape(self, ax, path, color='black', facecolor='none', linewidth=1, crs=ccrs.PlateCarree()):
162
+ '''
163
+
164
+ '''
165
+ reader = shpreader.Reader(path)
166
+
167
+ for record in reader.records():
168
+ geometry = record.geometry
169
+ ax.add_geometries([geometry], crs=crs, edgecolor=color, facecolor=facecolor, linewidth=linewidth)
170
+ plt.show(block=False)
171
+
172
+ def plot_mean(self, ax=None, xarray=None, block=False, multiplier=1, coastline=False, borders=False, colorbar=True,
173
+ colorbar_label=None, cmap='viridis', central_longitude=None, central_latitude=None, *args, **kwargs):
174
+ '''
175
+ Plots the mean behavior for the full time series
176
+ '''
177
+
178
+ if isinstance(ax, type(None)):
179
+ ax = self.create_plot(central_longitude)
180
+
181
+ if isinstance(xarray, type(None)):
182
+ xarray = self.to_xarray()
183
+ xarray = xarray.mean(dim=['production_datetime', 'ensemble_member', 'leadtime'])*multiplier
184
+
185
+
186
+ plot = xarray.plot.pcolormesh(ax=ax, transform=ccrs.PlateCarree(), x="lon", y="lat", add_colorbar=False, cmap=cmap, *args, **kwargs)
187
+ if colorbar:
188
+ if colorbar_label is None:
189
+ colorbar_label = self.units
190
+ cbar = plt.colorbar(plot, orientation='vertical', label=colorbar_label)
191
+ else:
192
+ cbar = None
193
+
194
+ if coastline:
195
+ ax.add_feature(cf.COASTLINE)
196
+ if borders:
197
+ ax.add_feature(cf.BORDERS)
198
+
199
+ plt.show(block=block)
200
+
201
+ return ax, cbar
202
+
203
+ def plot_mean_projected(self, ax=None, block=False, multiplier=1, *args, **kwargs):
204
+ '''
205
+ Plots the mean behavior for the full time series using the true coordinates
206
+ '''
207
+
208
+ sns.color_palette('viridis')
209
+
210
+ tmp = np.nanmean(self.data, axis=(0,1,2))*multiplier
211
+
212
+ if isinstance(ax, type(None)):
213
+ plt.figure()
214
+
215
+ plt.scatter(self.longitudes.ravel(), self.latitudes.ravel(), c=tmp.ravel(), *args, **kwargs)
216
+ plt.colorbar()
217
+ plt.show(block=block)
218
+
219
+ def plot_coordinates(self, ax=None, block=False, *args, **kwargs):
220
+ '''
221
+ Plots the available coordinates in the lat and lon matrixes
222
+ '''
223
+
224
+ if isinstance(ax, type(None)):
225
+ plt.figure()
226
+
227
+ plt.plot(self.longitudes.ravel(), self.latitudes.ravel(), 'k.', *args, **kwargs)
228
+ plt.show(block=block)
229
+
230
+ def plot_seasonal(self, lat, lon, window=31, leadtime=None, ax=None, block=False, multiplier=1, *args, **kwargs):
231
+
232
+ data = self.get_values_from_latlon(lat, lon)
233
+ if isinstance(leadtime, type(None)):
234
+ leadtime = self.leadtimes[0]
235
+
236
+ # Select specific leadtime columns (level 0 is leadtime)
237
+ data = data.xs(leadtime, level='leadtime', axis=1)
238
+
239
+ tmp = data.rolling(window=window, center=True).mean().dropna().stack().to_frame(name='values').reset_index()
240
+
241
+ # Use proper column name from index
242
+ idx_name = data.index.name if data.index.name else 'index'
243
+ if idx_name in tmp.columns:
244
+ tmp.rename(columns={idx_name: 'Production dates'}, inplace=True)
245
+
246
+ tmp.loc[:, 'Day of year'] = tmp.loc[:, 'Production dates'].dt.day_of_year
247
+
248
+ if isinstance(ax, type(None)):
249
+ plt.figure(figsize=(20,10))
250
+ ax = sns.lineplot(x='Day of year', y='values', data=tmp, errorbar=('pi', 95), ax=ax, *args, **kwargs)
251
+
252
+ return ax
253
+
254
+ def get_values_from_KML(self, kml, nameField=None, coverage_info=None, getCoverageInfo=False, elementwise=False):
255
+ '''
256
+ Returns a pandas table with averaged values
257
+ '''
258
+
259
+ if isinstance(coverage_info, type(None)):
260
+ cvr, columns, centroids = self.__coverageMatrixFromKML(kml, nameField)
261
+ coverage_info = (cvr, columns, centroids)
262
+ else:
263
+ cvr, columns, centroids = coverage_info
264
+
265
+ agg = self.__groupByMatrix(cvr, columns, elementwise=elementwise)
266
+
267
+ if getCoverageInfo:
268
+ return agg, centroids, coverage_info
269
+ else:
270
+ return agg, centroids
271
+
272
+ def get_quantiles_from_KML(self, kml, nameField=None, coverage_info=None, getCoverageInfo=False, resampling=None, precision=0.1, quantiles=[0.01, 0.1, 0.5, 0.9, 0.99]):
273
+ '''
274
+ Returns a pandas table with averaged values
275
+ '''
276
+
277
+ if isinstance(coverage_info, type(None)):
278
+ cvr, columns, centroids = self.__coverageMatrixFromKML(kml, nameField, normalize_by_area=False)
279
+ coverage_info = (cvr, columns, centroids)
280
+ else:
281
+ cvr, columns, centroids = coverage_info
282
+
283
+ agg = self.__groupByQuantile(cvr, columns, resampling, quantiles, precision)
284
+
285
+ if getCoverageInfo:
286
+ return agg, centroids, coverage_info
287
+ else:
288
+ return agg, centroids
289
+
290
+ def get_cropped(self, from_prod_date=dt.datetime(1900, 1, 1), to_prod_date=dt.datetime(2199, 12, 31),
291
+ from_lat=-90, to_lat=90, from_lon=-180, to_lon=180,
292
+ from_leadtime=None, to_leadtime=None):
293
+ '''
294
+ returns a MeteoRaster cropped in time and space
295
+ '''
296
+
297
+ self._diag('Cropping meteorology...', self.verbose)
298
+
299
+ date_idxs = np.arange(self.production_datetime.shape[0])[(self.production_datetime>=np.datetime64(from_prod_date)) & (self.production_datetime<=np.datetime64(to_prod_date))]
300
+
301
+ inside = np.zeros_like(self.latitudes).astype(bool)
302
+ inside[(np.round(self.longitudes,6)>=np.round(from_lon,6)) & (np.round(self.longitudes,6)<=np.round(to_lon,6)) & (np.round(self.latitudes,6)>=np.round(from_lat,6)) & (np.round(self.latitudes,6)<=np.round(to_lat,6))] = True
303
+ tmp = np.where(inside)
304
+ lat_idxs = np.arange(tmp[0].min(), tmp[0].max()+1)
305
+ lon_idxs = np.arange(tmp[1].min(), tmp[1].max()+1)
306
+
307
+ #=======================================================================
308
+ # lat_idxs = np.arange(self.latitudes.shape[0])[(self.latitudes>=from_lat) & (self.latitudes<=to_lat)]
309
+ # lon_idxs = np.arange(self.longitudes.shape[0])[(self.longitudes>=from_lon) & (self.longitudes<=to_lon)]
310
+ #=======================================================================
311
+
312
+ production_datetime = self.production_datetime[date_idxs]
313
+ latitudes = np.round(self.latitudes[lat_idxs, :],6)
314
+ latitudes = np.round(latitudes[:, lon_idxs],6)
315
+ longitudes = np.round(self.longitudes[lat_idxs, :],6)
316
+ longitudes = np.round(longitudes[:, lon_idxs],6)
317
+ data = self.data[date_idxs, :, :, :, :]
318
+ data = data[:, :, :, lat_idxs, :]
319
+ data = data[:, :, :, :, lon_idxs]
320
+
321
+ cropped = copy.deepcopy(self)
322
+ cropped.production_datetime = production_datetime
323
+ cropped.latitudes = latitudes
324
+ cropped.longitudes = longitudes
325
+
326
+ if from_leadtime:
327
+ idxs = cropped.leadtimes>=from_leadtime
328
+ cropped.leadtimes = cropped.leadtimes[idxs]
329
+ data = data[:, :, idxs, :, :]
330
+
331
+ if to_leadtime:
332
+ idxs = cropped.leadtimes<=to_leadtime
333
+ cropped.leadtimes = cropped.leadtimes[idxs]
334
+ data = data[:, :, idxs, :, :]
335
+
336
+ cropped.data = data
337
+
338
+ self._diag(' Done.', self.verbose)
339
+
340
+ return cropped
341
+
342
+ def trim(self):
343
+ '''
344
+
345
+ '''
346
+ with warnings.catch_warnings():
347
+ warnings.simplefilter("ignore", category=RuntimeWarning)
348
+ tmp = np.where(np.isfinite(np.nanmean(self.data, axis=(1, 2, 3, 4))))[0]
349
+ start = tmp[0]
350
+ end = tmp[-1]+1
351
+
352
+ self.data = self.data[start:end, ...]
353
+ self.production_datetime = self.production_datetime[start:end]
354
+
355
+ def join(self, meteoRaster, strickt=False, trim=False):
356
+ '''
357
+ joins a new meteoraster to self based on production dates
358
+ keeps a minimum number of ensembles.
359
+ keeps only matching leadtimes
360
+ '''
361
+
362
+ if trim:
363
+ self.trim()
364
+ meteoRaster.trim()
365
+
366
+ newData = meteoRaster.data
367
+ newproduction_datetime = meteoRaster.production_datetime
368
+ try:
369
+ newLeadtimes = meteoRaster.leadtimes.astype(self.leadtimes.dtype)
370
+ except Exception as ex:
371
+ newLeadtimes = meteoRaster.leadtimes
372
+ newLatitudes = meteoRaster.latitudes
373
+ newLongitudes = meteoRaster.longitudes
374
+
375
+
376
+ # handle spatial coverage
377
+ if (np.abs(self.latitudes-newLatitudes)>1E-10).any() or self.data.shape[-2]!=newData.shape[-2]:
378
+ raise(Exception('Latitudes do not match.'))
379
+
380
+ if (np.abs(self.longitudes-newLongitudes)>1E-10).any() or self.data.shape[-1]!=newData.shape[-1]:
381
+ raise(Exception('Longitudes do not match.'))
382
+
383
+
384
+ # handle ensembles
385
+ e0 = self.data.shape[1]
386
+ e1 = newData.shape[1]
387
+ warningMessage = 'The ensemble numbers do not match (base: %d, joint: %d). Filled missing with NaN.' % (e0, e1)
388
+ errorMessage = 'The ensemble numbers do not match (base: %d, joint: %d). Error issued in strickt mode.' % (e0, e1)
389
+ if e0>e1:
390
+ if strickt:
391
+ raise(Exception(errorMessage))
392
+ else:
393
+ warnings.warn(warningMessage)
394
+ shape = newData.shape
395
+ data = np.empty([shape[0], e0] + list(shape[2:])) * np.nan
396
+ data[:, :e1, :, :, :] = newData
397
+ newData = data
398
+ elif e0<e1:
399
+ if strickt:
400
+ raise(Exception(errorMessage))
401
+ else:
402
+ warnings.warn(warningMessage)
403
+ shape = self.data.shape
404
+ data = np.empty([shape[0], e1] + list(shape[2:])) * np.nan
405
+ data[:, :e0, :, :, :] = self.data
406
+ self.data = data
407
+
408
+
409
+ # handle leadtimes
410
+ v0 = np.where(np.isin(self.leadtimes, newLeadtimes))[0]
411
+ v1 = np.where(np.isin(newLeadtimes, self.leadtimes))[0]
412
+
413
+ if v0.size!=self.leadtimes.size or v1.size!=newLeadtimes.size:
414
+ if strickt:
415
+ raise(Exception('Leadtimes do not all match (base: %d, joint: %d). Error issued in strickt mode.' % (self.leadtimes.size, newLeadtimes.size)))
416
+ else:
417
+ warnings.warn('Leadtimes do not all match (base: %d, joint: %d). Keeping only matching values.' % (self.leadtimes.size, newLeadtimes.size))
418
+ self.leadtimes = self.leadtimes[v0]
419
+ self.data = self.data[:, :, v0, :, :]
420
+ newData = newData[:, :, v1, :, :]
421
+
422
+
423
+ # join
424
+ if self.leadtimes.shape != newLeadtimes.shape or not np.array_equal(self.leadtimes, newLeadtimes):
425
+ raise(Exception('Leadtimes do not match. Not implemented'))
426
+
427
+ if self.production_datetime[-1]<newproduction_datetime[0]:
428
+ # self if first
429
+ self.production_datetime = np.concatenate((self.production_datetime, newproduction_datetime))
430
+ self.data = np.concatenate((self.data, newData), axis=0)
431
+
432
+ elif self.production_datetime[0]>newproduction_datetime[-1]:
433
+ # joint is first
434
+ self.production_datetime = np.concatenate((newproduction_datetime, self.production_datetime))
435
+ self.data = np.concatenate((newData, self.data), axis=0)
436
+ else:
437
+ # mixed/interleaved production dates: align on the union of dates, fill NaN, new data overwrites on overlap
438
+ all_dates = np.sort(np.unique(np.concatenate((self.production_datetime, newproduction_datetime))))
439
+ target_shape = (all_dates.size,) + tuple(self.data.shape[1:])
440
+ target_dtype = np.result_type(self.data.dtype, np.float32)
441
+ merged = np.full(target_shape, np.nan, dtype=target_dtype)
442
+
443
+ # place existing data
444
+ idx_self = np.searchsorted(all_dates, self.production_datetime)
445
+ merged[idx_self, ...] = self.data.astype(target_dtype, copy=False)
446
+
447
+ # place new data (overwrite on overlap)
448
+ idx_new = np.searchsorted(all_dates, newproduction_datetime)
449
+ newData_cast = newData.astype(target_dtype, copy=False)
450
+ existing = merged[idx_new, ...]
451
+ nan_mask = np.isnan(existing)
452
+ if nan_mask.any():
453
+ existing[nan_mask] = newData_cast[nan_mask]
454
+ merged[idx_new, ...] = existing
455
+
456
+ self.production_datetime = all_dates
457
+ self.data = merged
458
+
459
+ def adjust_leadtimes(self, period='months'):
460
+ '''
461
+ Adjust leadtimes to relative time steps (useful for monthly)
462
+ '''
463
+ self._diag('Adjusting lead times...', self.verbose)
464
+
465
+ periodDefinition = {
466
+ 'months': pd.Series([pd.DateOffset(months=i) for i in range(1, math.ceil(pd.Series(self.leadtimes[-1]).dt.days.values[0]/30)+1)]),
467
+ }
468
+
469
+ if period not in periodDefinition.keys():
470
+ raise(Exception('%s is not a valid period.' % period))
471
+
472
+ warnings.warn('the function adjustLeadTimes() is experimental.')
473
+
474
+ times = self.production_datetime[0] + self.leadtimes
475
+ validTimes = pd.Timestamp(self.production_datetime[0]) + periodDefinition[period]
476
+
477
+ idxs = self.closestIdx(times, validTimes)
478
+ validPeriods = periodDefinition[period][:idxs.max()+1]
479
+
480
+ tmp = list(self.data.shape)
481
+ data = np.empty(tmp[:2] + [validPeriods.size] + tmp[-2:]) * np.nan
482
+
483
+ for i0 in idxs:
484
+ dataForLeadtime = self.data[:, :, idxs==i0, :, :]
485
+ finite = np.isfinite(dataForLeadtime)
486
+ count = np.sum(finite, axis=2)
487
+ if count.max()>1:
488
+ warnings.warn('Overlapping information detected when adjusting lead times (%s).' % str(validPeriods[i0]))
489
+
490
+ tmp = np.nanmean(dataForLeadtime, axis=2)
491
+ tmp[count==0] = np.nan
492
+
493
+ if np.isnan(tmp).all():
494
+ warnings.warn('Unused Lead time (%s)' % str(validPeriods[i0]))
495
+
496
+ data[:, :, i0, :, :] = tmp
497
+
498
+ self.data = data
499
+ if period=='months':
500
+ self.leadtimes = np.array([pd.DateOffset(months=i0.months-1) for i0 in validPeriods])
501
+
502
+
503
+ self._diag(' Done.', self.verbose)
504
+
505
+ def get_missing(self):
506
+ '''
507
+ Provides a diagnostic of missing data
508
+ '''
509
+
510
+ missingFraction = np.isnan(self.data).mean(axis=(-2, -1))
511
+ missingFraction = pd.DataFrame(missingFraction.reshape((missingFraction.shape[0], np.prod(missingFraction.shape[1:]))), index=self.production_datetime)
512
+ missingFraction.index.name = 'production_datetime'
513
+ missingFraction.columns = pd.MultiIndex.from_product((self.leadtimes, np.arange(self.data.shape[1])), names=['leadtime', 'ensemble_member'])
514
+
515
+ return missingFraction.transpose()
516
+
517
+ def get_values_from_latlon(self, lat, lon):
518
+ '''
519
+ Returns a dataframe with data from the pixel closest to the desired point
520
+ '''
521
+
522
+ idxLat, idxLon = self._getClosestPixels(lat, lon)
523
+
524
+ if idxLat.size>1 or idxLon.size>1:
525
+ warnings.warn('Several pixels are "closest". Averaging results.')
526
+ tmp = []
527
+ for iLat, iLon in zip(idxLat, idxLon):
528
+ tmp.append(np.expand_dims(self.data[:, :, :, iLat, iLon], axis=0))
529
+ tmp = np.concatenate(tmp, axis=0)
530
+ tmp = np.nanmean(tmp, axis=0) # [production_datetime, ensemble_member, leadtimes]
531
+
532
+ data = pd.DataFrame(tmp.reshape(tmp.shape[0]*tmp.shape[1], tmp.shape[2]))
533
+ data.index = pd.MultiIndex.from_product((self.production_datetime.ravel(), np.arange(self.data.shape[1])), names=['production_datetime', 'ensemble_member'])
534
+ data.columns = self.leadtimes
535
+ data.columns.name = 'leadtime'
536
+
537
+ data = data.unstack('ensemble_member')
538
+
539
+ return data
540
+
541
+ @staticmethod
542
+ def get_values_from_latlon_by_event(production_date_dataframe):
543
+ '''
544
+ !!!!! Change the index name to Event dates
545
+ '''
546
+
547
+ ensembles = production_date_dataframe.columns.get_level_values('ensemble_member').unique()
548
+ full_data_ = []
549
+ for ensemble in ensembles:
550
+ tmp = production_date_dataframe.loc[:, production_date_dataframe.columns.get_level_values('ensemble_member')==ensemble]
551
+ data_ = []
552
+ for i0 in range(tmp.shape[1]):
553
+ tmp_ = tmp.iloc[:, [i0]]
554
+ tmp_.index = tmp_.index + tmp_.columns.get_level_values('leadtime')[0]
555
+ data_.append(tmp_)
556
+ full_data_.append(pd.concat(data_, axis=1))
557
+
558
+ return pd.concat(full_data_, axis=1)
559
+
560
+ def plot_availability(self, missing=None, individualMembers=False):
561
+ '''
562
+ Plots the availability of the data
563
+ '''
564
+
565
+ if isinstance(missing, type(None)):
566
+ missing = self.get_missing()
567
+
568
+ available = missing < 0.95
569
+
570
+ if individualMembers:
571
+ tmp = available.stack().to_frame()
572
+ else:
573
+ virtualLevels = pd.to_datetime(self.production_datetime[0]) + self.leadtimes
574
+ available.index = available.index.set_levels(virtualLevels, level=0)
575
+ tmp = available.groupby(level=0).sum().transpose().stack().to_frame()
576
+ tmp.index = tmp.index.set_levels(self.leadtimes, level=1)
577
+
578
+
579
+ tmp.columns = ['available']
580
+ tmp = tmp.loc[tmp['available']>0,:].reset_index()
581
+ tmp.loc[:, 'event_dates'] = tmp.loc[:, 'production_datetime'] + tmp.loc[:, 'leadtime']
582
+
583
+ sns.set_theme(style="whitegrid")
584
+ if individualMembers:
585
+ g = sns.relplot(
586
+ data=tmp,
587
+ x='event_dates', y='production_datetime', hue='ensemble_member',
588
+ palette='plasma',
589
+ )
590
+ else:
591
+ g = sns.relplot(
592
+ data=tmp,
593
+ x='event_dates', y='production_datetime', hue='available', size='available',
594
+ palette='viridis',
595
+ sizes={s: s*10 for s in np.arange(1,max((11, tmp.loc[:, 'available'].max()+1)))},
596
+ #===================================================================
597
+ # size_order=np.arange(1, max((11, tmp.loc[:, 'available'].max()+1)))
598
+ #===================================================================
599
+ )
600
+ g.ax.xaxis.grid(True, "minor", linewidth=.25)
601
+ g.ax.yaxis.grid(True, "minor", linewidth=.25)
602
+
603
+ plt.show(block=False)
604
+
605
+ def resample_timestep(self, rule, fun=np.mean):
606
+ '''
607
+ Resamples data according to pandas conventions
608
+ https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.resample.html
609
+
610
+ rule='MS' resamples to the start of the month
611
+ '''
612
+
613
+ warnings.warn('the function resample_timestep() is experimental.')
614
+
615
+ self._diag(' Resampling time step (rule: %s, %s)...' % (rule, str(fun)), self.verbose)
616
+
617
+ # prepare a template dataframe
618
+ tmp = pd.DataFrame(np.arange(self.production_datetime.shape[0]), index=self.production_datetime)
619
+ idxMax = tmp.resample(rule=rule).max().astype(int)+1
620
+
621
+ resampled = np.empty([idxMax.shape[0]] + list(self.data.shape[1:])) * np.nan
622
+ idx0 = 0
623
+ for i0, i1 in enumerate(idxMax.values.ravel()):
624
+ resampled[i0, :, :, :, :] = fun(self.data[idx0:i1, :, :, :, :], axis=0, keepdims=True)
625
+ idx0 = i1
626
+
627
+ self.data = resampled
628
+ self.production_datetime = idxMax.index
629
+
630
+ self._diag(' Done (%s to %s).' % (self.production_datetime[0].strftime('%Y-%m-%d %H:%M:%S', self.verbose),
631
+ self.production_datetime[-1].strftime('%Y-%m-%d %H:%M:%S')))
632
+
633
+ def is_complete(self, full_ensemble:bool=True, space_completeness:bool=False) -> bool:
634
+ '''
635
+ Checks if the dataset is complete.
636
+
637
+ - Always requires every production_datetime x leadtime slice to have finite data.
638
+ - If full_ensemble is True, all ensemble members must be finite for each slice;
639
+ otherwise, at least one ensemble member with finite data is enough.
640
+ - If space_completeness is True, all spatial points (y, x) must be finite;
641
+ otherwise, at least one spatial point with finite data is enough.
642
+
643
+ :param full_ensemble: Require all ensemble members to be complete.
644
+ :param space_completeness: Require full spatial coverage.
645
+ :return: True if complete, False otherwise.
646
+ '''
647
+ if self.data is None:
648
+ return False
649
+
650
+ finite = np.isfinite(self.data)
651
+
652
+ # Ensemble requirement
653
+ if full_ensemble:
654
+ finite = finite.all(axis=1) # -> [prod, leadtime, y, x]
655
+ else:
656
+ finite = finite.any(axis=1) # -> [prod, leadtime, y, x]
657
+
658
+ # Spatial requirement
659
+ if space_completeness:
660
+ finite = finite.all(axis=(-2, -1)) # -> [prod, leadtime]
661
+ else:
662
+ finite = finite.any(axis=(-2, -1)) # -> [prod, leadtime]
663
+
664
+ # Temporal requirement: every production_datetime and every leadtime must be complete
665
+ return finite.all()
666
+
667
+ def get_complete_index(self, full_ensemble:bool=True, space_completeness:bool=False) -> pd.DataFrame:
668
+ '''
669
+
670
+ '''
671
+
672
+ self._diag('Retrieving complete index...', self.verbose)
673
+
674
+ finite = np.isfinite(self.data)
675
+
676
+ # Ensemble requirement
677
+ if full_ensemble:
678
+ finite = finite.all(axis=1) # -> [production_datetime, leadtime, y, x]
679
+ else:
680
+ finite = finite.any(axis=1) # -> [production_datetime, leadtime, y, x]
681
+
682
+ # Spatial requirement
683
+ if space_completeness:
684
+ finite = finite.all(axis=(-2, -1)) # -> [production_datetime, leadtime] or [production_datetime, ensemble_member, leadtime]
685
+ else:
686
+ finite = finite.any(axis=(-2, -1)) # -> [production_datetime, leadtime] or [production_datetime, ensemble_member, leadtime]
687
+
688
+ completeness_index = pd.DataFrame(finite, index=self.production_datetime, columns=self.leadtimes)
689
+ completeness_index.index.name = 'production_datetime'
690
+ completeness_index.columns.name = 'leadtime'
691
+
692
+ return completeness_index
693
+
694
+ @classmethod
695
+ def get_completeness(cls, file) -> bool:
696
+ '''
697
+ Docstring for check_completeness
698
+
699
+ :param file: Description
700
+ :return: Description
701
+ :rtype: bool
702
+ '''
703
+
704
+ try:
705
+ with xr.open_dataset(Path(file), decode_times=False, engine=cls.ENGINE) as ds:
706
+ return ds.attrs['complete'].lower() in ['true', 'yes']
707
+ except Exception as ex:
708
+ raise(ex)
709
+
710
+ def save(self, file, complevel=1, complete=None) -> None:
711
+ '''
712
+ Saves a netcdf file (.nc)
713
+ '''
714
+
715
+ if not isinstance(file, Path):
716
+ file = Path(file)
717
+
718
+ self._diag('Saving %s...' % file, self.verbose)
719
+
720
+ data_array = self.to_xarray()
721
+ ds = xr.Dataset({self.variable: data_array})
722
+ ds['lat'].attrs['standard_name'] = 'latitude'
723
+ ds['lon'].attrs['standard_name'] = 'longitude'
724
+
725
+ ds.attrs = {k: v for k, v in self.ds_attrs.items()}
726
+ ds.attrs['history'] = data_array.attrs['saved']
727
+ if complete is None:
728
+ ds.attrs['complete'] = str(self.is_complete())
729
+ else:
730
+ ds.attrs['complete'] = str(complete)
731
+
732
+ tmp_file = None
733
+ try:
734
+ with tempfile.NamedTemporaryFile(delete=False) as tmp:
735
+ tmp_file = Path(tmp.name)
736
+
737
+ if complevel>0:
738
+ encoding = {
739
+ self.variable: {"zlib": True, "complevel": complevel},
740
+ }
741
+ ds.to_netcdf(tmp_file, encoding=encoding, engine=self.ENGINE)
742
+ else:
743
+ ds.to_netcdf(tmp_file, engine=self.ENGINE)
744
+
745
+ shutil.move(tmp_file, file)
746
+ except Exception:
747
+ if tmp_file and tmp_file.exists():
748
+ try:
749
+ tmp_file.unlink()
750
+ except Exception:
751
+ pass
752
+ raise
753
+
754
+ self._diag(' Saved.', self.verbose)
755
+
756
+ def to_xarray(self) -> xr.DataArray:
757
+ '''
758
+ Docstring for to_xarray
759
+
760
+ :param self: Description
761
+ :return: Description
762
+ :rtype: DataArray
763
+ '''
764
+
765
+ leadtimes = self.leadtimes
766
+ leadtime_format = 'pd.Timedelta'
767
+ if isinstance(leadtimes[-1], pd.DateOffset):
768
+ if hasattr(leadtimes[-1], 'months'):
769
+ leadtimes = [i.months for i in leadtimes]
770
+ leadtime_format = 'pd.DateOffset:months'
771
+ else:
772
+ leadtimes = [i.years for i in leadtimes]
773
+ leadtime_format = 'pd.DateOffset:years'
774
+
775
+ data_array = xr.DataArray(
776
+ data=self.data,
777
+ dims=['production_datetime', 'ensemble_member', 'leadtime', 'y', 'x'],
778
+ coords={
779
+ 'production_datetime': self.production_datetime, # pd.Timestamp
780
+ 'ensemble_member': np.arange(self.data.shape[1], dtype=int), # int
781
+ 'leadtime': leadtimes, # pd.Timedelta / DateOffset
782
+ 'lat': (['y', 'x'], self.latitudes), # float
783
+ 'lon': (['y', 'x'], self.longitudes), # float
784
+ 'y': np.arange(self.data.shape[-2], dtype=int),
785
+ 'x': np.arange(self.data.shape[-1], dtype=int),
786
+ 'leadtime_format': leadtime_format,
787
+ },
788
+ name=self.variable,
789
+ attrs={'units': self.units, 'variable': self.variable, 'saved': f'Saved by {self.__class__.__name__} v{self.VERSION} @ {pd.Timestamp.now().isoformat()}'},
790
+ )
791
+
792
+ return data_array
793
+
794
+ @classmethod
795
+ def load(cls, file, verbose=None) -> "MeteoRaster":
796
+ '''
797
+ Loads a MeteoRaster from a NetCDF (.nc / .nct) file written by save().
798
+ '''
799
+
800
+ if not isinstance(file, Path):
801
+ file = Path(file)
802
+
803
+ kwargs = {}
804
+ if verbose is None:
805
+ verbose = cls.VERBOSE
806
+ else:
807
+ kwargs['verbose'] = verbose
808
+ cls._diag('Loading %s...' % file, verbose)
809
+
810
+ if file.suffix.lower() not in {'.nc', '.nct'}:
811
+ raise ValueError('Only NetCDF files written by save() are supported.')
812
+
813
+ with xr.open_dataset(file, engine=cls.ENGINE) as ds:
814
+ # pick the first data variable if name is unknown
815
+ var_name = cls.__detect_variable_name(ds)
816
+ da = ds[var_name]
817
+
818
+ units = da.attrs.get('units', 'unknown')
819
+ variable = da.attrs.get('variable', 'unknown')
820
+ da_attrs = ds.attrs
821
+ ds_attrs = ds.attrs
822
+
823
+ leadtimes = ds['leadtime'].values
824
+ if hasattr(ds['leadtime'], 'leadtime_format'):
825
+ leadtime_format = str(ds['leadtime'].leadtime_format.values)
826
+ if leadtime_format.split(':')[0]=='pd.DateOffset':
827
+ leadtimes = pd.Index([pd.DateOffset(**{leadtime_format.split(':')[1]: lt}) for lt in leadtimes])
828
+
829
+ meteo = cls(
830
+ data=da.values,
831
+ latitudes=ds['lat'].values,
832
+ longitudes=ds['lon'].values,
833
+ production_datetime=ds['production_datetime'].values,
834
+ leadtimes=leadtimes,
835
+ units=units,
836
+ variable=variable,
837
+ da_attrs=da_attrs,
838
+ ds_attrs=ds_attrs,
839
+ **kwargs,
840
+ )
841
+
842
+ cls._diag(' Loaded.', verbose)
843
+
844
+ return meteo
845
+
846
+ @staticmethod
847
+ def __detect_variable_name(ds: xr.Dataset) -> str:
848
+ # Require single-variable datasets written by save()
849
+ if len(ds.data_vars) != 1:
850
+ raise ValueError('NetCDF file must contain exactly one data variable written by save().')
851
+ return next(iter(ds.data_vars))
852
+
853
+ def copy(self):
854
+ '''
855
+ Makes a copy of a MeteoRaster file
856
+ '''
857
+
858
+ mr = MeteoRaster(data=self.data.copy(),
859
+ latitudes=self.latitudes.copy(), longitudes=self.longitudes.copy(),
860
+ production_datetime=self.production_datetime.copy(), leadtimes=self.leadtimes.copy(),
861
+ da_attrs=self.da_attrs, ds_attrs=self.ds_attrs,
862
+ )
863
+ return mr
864
+
865
+ def _getClosestPixels(self, lat, lon):
866
+ '''
867
+ Returns the indexes of the pixel closer to a point
868
+ '''
869
+
870
+ dist = ((self.longitudes-lon)**2+(self.latitudes-lat)**2)**0.5
871
+ idx_lat, idx_lon = np.where(dist==dist.ravel().min())
872
+
873
+ return (idx_lat, idx_lon)
874
+
875
+ @staticmethod
876
+ def closestIdx(X, Y):
877
+ '''
878
+ Returns the list of Y indexes closest the values of X
879
+ Dimension of X.
880
+ Indexes to be used in Y.
881
+ '''
882
+
883
+ idxs = np.empty_like(X, dtype=int)
884
+ for i0, x0 in enumerate(X):
885
+ idxs[i0] = np.argmin(np.abs(x0-np.array(Y)))
886
+
887
+ return idxs
888
+
889
+ def _flipLatitude(self):
890
+ '''
891
+ flips the latitude of the raster
892
+ '''
893
+
894
+ self.data = np.flip(self.data, axis=3)
895
+ self.latitudes = np.flip(self.latitudes, axis=0)
896
+ self.longitudes = np.flip(self.longitudes, axis=0)
897
+
898
+ self._diag('Flipped latitudes.', self.verbose)
899
+
900
+ def _fixStartAtGreenwich(self):
901
+ '''
902
+ modifies rasters with longitudes starting at Greenwich to the more conveninent -180E.
903
+ '''
904
+
905
+ westIndexes = np.arange(len(self.longitudes))[self.longitudes>180]
906
+ eastIndexes = np.arange(len(self.longitudes))[self.longitudes<=180]
907
+
908
+ longitudes = np.concatenate((self.longitudes[westIndexes]-360, self.longitudes[eastIndexes]))
909
+
910
+ westData = self.data[:, :, : ,: ,westIndexes]
911
+ eastData = self.data[:, :, : ,: ,eastIndexes]
912
+
913
+ data = np.concatenate((westData, eastData), axis=-1)
914
+
915
+ self.longitudes = longitudes
916
+ self.data = data
917
+
918
+ self._diag('Fixed start at Greenwich.', self.verbose)
919
+
920
+ def __groupByMatrix(self, coverage, columns, elementwise=False):
921
+ '''
922
+ Groups the data by a coverage matrix (see groupByKML for matrix definition)
923
+ '''
924
+
925
+ self._diag('Executing spatial aggregation...', self.verbose)
926
+
927
+ #Verification of valid tiles (assumes first coverage zone to be representative (for speed and memory)
928
+ val_ = np.isfinite(self.data).astype(bool)
929
+ cvr = np.tile(coverage[:, :, :, :, :, 0].astype(np.single), list(val_.shape[:(val_.ndim-2)]) + [1, 1])
930
+ val = np.nansum(cvr*val_, axis=(-2, -1))>0.8
931
+ val = np.tile(np.expand_dims(val,-1), [1]*val.ndim + [coverage.shape[-1]])
932
+
933
+ if elementwise:
934
+ agg = np.empty(list(self.data.shape[:3]) + [len(columns)]).astype(np.single) * np.nan
935
+
936
+ data_single = self.data.astype(np.single)
937
+ coverage_single = coverage.astype(np.single)
938
+
939
+ for i0 in range(len(columns)):
940
+ agg[:, :, :, i0] = np.nansum(data_single * np.tile(coverage_single[:, :, :, :, :, i0], list(data_single.shape[:(self.data.ndim-2)]) + [1, 1]), axis=(-2, -1))
941
+ self._diag(' % 5.1f%%' % ((i0+1)/len(columns)*100), self.verbose)
942
+ else:
943
+ #Creation of matrices for calculation
944
+ dat = np.tile(np.expand_dims(self.data.astype(np.single), axis=-1), [1]*self.data.ndim + [coverage.shape[-1]])
945
+ cvr = np.tile(coverage.astype(np.single), list(dat.shape[:(dat.ndim-3)]) + [1, 1, 1])
946
+
947
+ #Correction of the coveragefor missing data
948
+ dat_finite = np.isfinite(dat).astype(bool)
949
+ cvr_finite = cvr.copy()
950
+ cvr_finite[~dat_finite] = 0
951
+ weight_update = 1 / cvr_finite.sum(axis=(-3, -2))
952
+ weight_update = np.tile(weight_update[:, :, :, np.newaxis, np.newaxis, :], [1, 1, 1] + list(cvr.shape[-3:-1]) + [1])
953
+
954
+ dat *= cvr
955
+ dat *= weight_update
956
+
957
+ cvr = None
958
+ gc.collect()
959
+
960
+ #Calculation with garbage collection
961
+ agg = np.nansum(dat, axis=(-3, -2))
962
+ dat = None
963
+ gc.collect()
964
+
965
+ #Mark invalid tiles as NaN
966
+ agg[~val] = np.nan
967
+
968
+ tmp = np.reshape(agg, (agg.shape[0], np.prod(agg.shape[1:])), order='F')
969
+ agg = pd.DataFrame(tmp, index=self.production_datetime.ravel())
970
+ agg.columns = columns
971
+ agg.index.name = 'Production date'
972
+
973
+ self._diag(' Done.', self.verbose)
974
+
975
+ return agg
976
+
977
+ def __groupByQuantile(self, coverage, columns, resampling, quantiles, precision=0.1):
978
+ '''
979
+ Groups the data by quantiles in time and space
980
+
981
+ precision should be between 0 and 1.
982
+ computation time will be proportional to 1/precision
983
+ '''
984
+
985
+ self._diag('Executing spatial aggregation...', self.verbose)
986
+
987
+ if not resampling is None:
988
+ resampled_idx = pd.DataFrame(self.production_datetime, self.production_datetime).resample(resampling).indices
989
+ time_idx = list(resampled_idx.keys())
990
+ time_groups = [i for _, i in resampled_idx.items()]
991
+ else:
992
+ time_idx = self.production_datetime
993
+ time_groups = [[i] for i, _ in enumerate(time_idx)]
994
+
995
+ #Creation of matrices for calculation
996
+ dat = np.tile(np.expand_dims(self.data.astype(np.single), axis=-1), [1]*self.data.ndim + [coverage.shape[-1]])
997
+ coverage_ = coverage.astype(np.single)
998
+ coverage_[coverage_==0] = np.nan
999
+ gc.collect()
1000
+
1001
+ #Calculation with garbage collection
1002
+ agg_ = []
1003
+ for g0 in time_groups:
1004
+ data = dat[g0,:, :, :, :].copy()
1005
+
1006
+ # Verification of valid tiles (assumes first coverage zone to be representative (for speed and memory)
1007
+ cvr = np.tile(coverage_, list(data.shape[:(dat.ndim-3)]) + [1, 1, 1])
1008
+ val = cvr.copy()
1009
+ val[val<=0] = np.nan
1010
+ val[val>0] = 1
1011
+
1012
+ data_ = data * val
1013
+
1014
+ folds = np.round(cvr/precision)
1015
+ n_folds = int(np.nanmax(folds))
1016
+ data_unfolded = np.tile(np.expand_dims(data, 0), [n_folds] + [1] * data.ndim) * np.nan
1017
+ tmp = data_.copy()
1018
+ for i1 in range(n_folds):
1019
+ tmp[folds<i1+1] = np.nan
1020
+ data_unfolded[i1,:,:,:,:,:,:] = tmp
1021
+
1022
+ # Check what pixels are valid (more than 80% values available)
1023
+ val = np.nansum(np.isfinite(data_), axis=(0, -3, -2)) / np.nansum(np.isfinite(val), axis=(0, -3, -2)) > 0.8
1024
+ val = np.tile(np.expand_dims(val, (0, -1)), [1]*(val.ndim + 1) + [len(quantiles)])
1025
+
1026
+ # Calculate the quantiles
1027
+ raise(Exception('This must be reviewed.'))
1028
+ tmp = np.rollaxis(np.nanquantile(np.expand_dims(data_unfolded, 0), quantiles, axis=(1, 2, -3, -2)), 0, len(quantiles)) #[time, lead, ensemble, zones, quantiles]
1029
+ tmp[~val] = np.nan
1030
+ agg_.append(tmp)
1031
+
1032
+ agg__ = np.r_[agg_]
1033
+
1034
+ tmp = np.reshape(agg__, (agg__.shape[0], np.prod(agg__.shape[1:])), order='F')
1035
+ agg = pd.DataFrame(tmp, index=time_idx)
1036
+
1037
+ tmp = pd.concat([columns.to_frame()]*len(quantiles))
1038
+ tmp.loc[:, 'Quantile'] = np.array([[q]*np.prod(agg__.shape[1:-1]) for q in quantiles]).ravel()
1039
+ agg.columns = pd.MultiIndex.from_frame(tmp)
1040
+ agg.index.name = 'Production date'
1041
+ agg.sort_index(axis=1, inplace=True)
1042
+
1043
+ #=======================================================================
1044
+ # agg.to_clipboard(excel=True)
1045
+ #=======================================================================
1046
+
1047
+ self._diag(' Done.', self.verbose)
1048
+
1049
+ return agg
1050
+
1051
+ def __coverageMatrixFromKML(self, kml, nameField=None, buffer=2, normalize_by_area=True):
1052
+ '''
1053
+ Creates a coverage matrix from polygons contained in a kml file
1054
+ '''
1055
+
1056
+ self._diag('Creating a coverage matrix for spatial aggregation...', self.verbose)
1057
+
1058
+ # Correct kml file if required
1059
+ self.__correctKMLHeader(kml)
1060
+
1061
+ # Read geometry reference KML file
1062
+ ids = []
1063
+ coordinates = []
1064
+ document = et.parse(kml).getroot().find('Document')
1065
+ for i0, placemark in enumerate(document.findall('.//Placemark')):
1066
+ #https://docs.python.org/3/library/xml.etree.elementtree.html
1067
+ try:
1068
+ if not isinstance(nameField, type(None)):
1069
+ id0 = placemark.find(".//SimpleData[@name='%s']" % nameField).text
1070
+ else:
1071
+ id0 = str(i0)
1072
+ outer = [pd.Series(c.text.replace(' \n', '').replace('\n', '').replace('\t', '').split(' ')).str.split(',', n=2, expand=True) for c in placemark.findall('.//Polygon/outerBoundaryIs//coordinates')]
1073
+ outer = [c.astype(float) for c in outer]
1074
+ inner = [pd.Series(c.text.replace(' \n', '').replace('\n', '').replace('\t', '').split(' ')).str.split(',', n=2, expand=True) for c in placemark.findall('.//Polygon/innerBoundaryIs//coordinates')]
1075
+ inner = [c.astype(float) for c in inner]
1076
+ ids.append(id0)
1077
+ coordinates.append({'outer': outer, 'inner': inner})
1078
+ except Exception as ex:
1079
+ print(str(ex))
1080
+
1081
+ cvr = np.zeros([1, 1, 1] + list(self.data.shape[3:]) + [len(ids)], dtype=np.single) # the last coordinate of the cvr represents the polygon
1082
+
1083
+ longitudes = self.longitudes
1084
+ latitudes = self.latitudes
1085
+ idx_lat_ = np.tile(np.expand_dims(np.arange(longitudes.shape[0]),axis=1),(1, longitudes.shape[1])).ravel()
1086
+ idx_lon_ = np.tile(np.arange(longitudes.shape[1]),(longitudes.shape[0], 1)).ravel()
1087
+ lonInterpolator = interp.LinearNDInterpolator(np.array([idx_lat_, idx_lon_]).T, longitudes.ravel()) # f(lat_id, lon_id)
1088
+ latInterpolator = interp.LinearNDInterpolator(np.array([idx_lat_, idx_lon_]).T, latitudes.ravel()) # f(lat_id, lon_id)
1089
+
1090
+ # Create GDAL geometry
1091
+ centroids = {'x':[], 'y':[]}
1092
+ for i0, coords in enumerate(coordinates):
1093
+ try:
1094
+ # Add all "positive" rings
1095
+ poly = ogr.Geometry(ogr.wkbPolygon)
1096
+ for o1 in coords['outer']:
1097
+ for r0 in o1:
1098
+ polyTmp = ogr.Geometry(ogr.wkbPolygon)
1099
+ ring = ogr.Geometry(ogr.wkbLinearRing)
1100
+ for row in o1.iterrows():
1101
+ ring.AddPoint(row[1][0], row[1][1])
1102
+ _ = polyTmp.AddGeometry(ring)
1103
+ poly = poly.Union(polyTmp)
1104
+
1105
+ # Join all "negative rings
1106
+ polyHoles = ogr.Geometry(ogr.wkbPolygon)
1107
+ for o1 in coords['inner']:
1108
+ for r0 in o1:
1109
+ polyTmp = ogr.Geometry(ogr.wkbPolygon)
1110
+ ring = ogr.Geometry(ogr.wkbLinearRing)
1111
+ for row in o1.iterrows():
1112
+ ring.AddPoint(row[1][0], row[1][1])
1113
+ _ = polyTmp.AddGeometry(ring)
1114
+ polyHoles = polyHoles.Union(polyTmp)
1115
+
1116
+ # Obtain the final polygon
1117
+ poly = poly.SymmetricDifference(polyHoles)
1118
+
1119
+ # Select pixel subset
1120
+ #===============================================================
1121
+ # plt.figure(); plt.imshow(longitudes); plt.colorbar()
1122
+ # plt.figure(); plt.imshow(latitudes); plt.colorbar()
1123
+ # plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=self.data.squeeze().mean(axis=0).ravel()); plt.colorbar()
1124
+ # plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=longitudes.ravel()); plt.colorbar()
1125
+ # plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=latitudes.ravel()); plt.colorbar()
1126
+ #===============================================================
1127
+
1128
+ extent = poly.Buffer(buffer).GetEnvelope() #wesn
1129
+ inside = np.zeros_like(latitudes).astype(bool)
1130
+ inside[(longitudes>=extent[0]) & (longitudes<=extent[1]) & (latitudes>=extent[2]) & (latitudes<=extent[3])] = True
1131
+ tmp = np.where(inside)
1132
+ lat_idxs = np.arange(tmp[0].min()-1, tmp[0].max()+1)
1133
+ lat_idxs = lat_idxs[(lat_idxs>0) & (lat_idxs<latitudes.shape[0]-1)]
1134
+ lon_idxs = np.arange(tmp[1].min()-1, tmp[1].max()+1)
1135
+ lon_idxs = lon_idxs[(lon_idxs>0) & (lon_idxs<longitudes.shape[1]-1)]
1136
+
1137
+ #===============================================================
1138
+ # plt.figure(); plt.imshow(inside); plt.colorbar()
1139
+ # plt.figure(); plt.scatter(longitudes.ravel(), latitudes.ravel(),c=inside.ravel()); plt.colorbar()
1140
+ #===============================================================
1141
+
1142
+ #===============================================================
1143
+ # extent = [extent[0]-hdLon*2, extent[1]+hdLon*2, extent[2]-hdLat*2, extent[3]+hdLat*2]
1144
+ # lon_idxs = np.arange(self.longitudes.shape[0])[(self.longitudes>=extent[0]) & (self.longitudes<=extent[1])]
1145
+ # lat_idxs = np.arange(self.latitudes.shape[0])[(self.latitudes>=extent[2]) & (self.latitudes<=extent[3])]
1146
+ #===============================================================
1147
+
1148
+ x, y, _ = poly.Centroid().GetPoint()
1149
+ centroids['x'].append(x)
1150
+ centroids['y'].append(y)
1151
+
1152
+ for i2 in lon_idxs:
1153
+ for i3 in lat_idxs:
1154
+ #=======================================================
1155
+ # lon = self.longitudes[i3, i2]
1156
+ # lat = self.latitudes[i3, i2]
1157
+ # plt.plot(lon, lat, 'rx')
1158
+ # plt.plot(lonInterpolator(i3-0.5,i2+0.5), latInterpolator(i3-0.5,i2+0.5), '.m')
1159
+ #=======================================================
1160
+
1161
+ points = []
1162
+ points.append((float(lonInterpolator(i3-0.5, i2+0.5)), float(latInterpolator(i3-0.5, i2+0.5))))
1163
+ points.append((float(lonInterpolator(i3+0.5, i2+0.5)), float(latInterpolator(i3+0.5, i2+0.5))))
1164
+ points.append((float(lonInterpolator(i3+0.5, i2-0.5)), float(latInterpolator(i3+0.5, i2-0.5))))
1165
+ points.append((float(lonInterpolator(i3-0.5, i2-0.5)), float(latInterpolator(i3-0.5, i2-0.5))))
1166
+ points.append((float(lonInterpolator(i3-0.5, i2+0.5)), float(latInterpolator(i3-0.5, i2+0.5))))
1167
+
1168
+ ring = ogr.Geometry(ogr.wkbLinearRing)
1169
+ for p in points:
1170
+ ring.AddPoint(*p)
1171
+ #===================================================
1172
+ # plt.plot(*p,'.b')
1173
+ #===================================================
1174
+
1175
+ pixel = ogr.Geometry(ogr.wkbPolygon)
1176
+ _ = pixel.AddGeometry(ring)
1177
+
1178
+ intersection = pixel.Intersection(poly)
1179
+ area = intersection.Area()
1180
+
1181
+ if area>0:
1182
+ cvr[0, 0, 0, i3, i2, i0] = area
1183
+ except Exception as ex:
1184
+ raise(ex)
1185
+
1186
+ # Define columns (drops redundant levels (except zone))
1187
+ columns = pd.MultiIndex.from_product((ids, self.leadtimes, 1+np.arange(self.data.shape[self.ENSEMBLEMEMBERpOSITION])), names=('zone', 'leadtime', 'ensemble_member'))
1188
+ drop = []
1189
+ for i0, (n0, l0) in enumerate(zip(columns.names, list(columns.levels))):
1190
+ if l0.shape[0]==1 and n0!='Zone':
1191
+ self._diag(' Level %s is redundant and will be dropped.' % n0, self.verbose)
1192
+ drop.append(i0)
1193
+ for i0 in drop[::-1]:
1194
+ columns = columns.droplevel(i0)
1195
+
1196
+ # Normalize
1197
+ if normalize_by_area:
1198
+ areas = cvr.sum(axis=(0, 1, 2, 3, 4))
1199
+ cvr /= np.tile(areas, list(cvr.shape[:-1]) + [1])
1200
+ else:
1201
+ max_pixel_area = cvr.max(axis=tuple(np.arange(cvr.ndim-1)))
1202
+ cvr /= np.tile(max_pixel_area, list(cvr.shape[:-1]) + [1])
1203
+
1204
+ # Prepare centroids for export
1205
+ centroids = pd.DataFrame(centroids, index=ids)
1206
+
1207
+ self._diag(' Done.', self.verbose)
1208
+
1209
+ return cvr, columns, centroids
1210
+
1211
+ def __correctKMLHeader(self, kml, encoding='utf-8'):
1212
+ '''
1213
+ Corrects the header of a KML file to that it can be read for model data preparation
1214
+ '''
1215
+
1216
+ modificationRequired = False
1217
+ line = ''
1218
+ with open(kml, 'r', encoding=encoding) as file:
1219
+ for line in file:
1220
+ if line.strip().startswith('<kml'):
1221
+ if line.strip() != '<kml>':
1222
+ modificationRequired = True
1223
+ break
1224
+
1225
+ if modificationRequired:
1226
+ if not isinstance(kml, str):
1227
+ kml = str(kml)
1228
+
1229
+ tmpKML = kml.replace('.kml', '__tmp.kml')
1230
+ with open(kml, 'r', encoding=encoding) as file_in:
1231
+ with open(tmpKML, 'w', encoding=encoding) as file_out:
1232
+ for line in file_in:
1233
+ if line.strip().startswith('<kml'):
1234
+ if line.strip() != '<kml>':
1235
+ line = '<kml>\n'
1236
+ __ = file_out.write(line)
1237
+ os.remove(kml)
1238
+ os.rename(tmpKML, kml)
1239
+
1240
+ @staticmethod
1241
+ def _diag(message, show=True):
1242
+ if show:
1243
+ print(message)
1244
+
1245
+ def resampleTimeStep(self, *args, **kwargs):
1246
+ warnings.warn('resampleTimeStep() is deprecated and will be removed in a future release; call resample_timestep() directly instead.', DeprecationWarning, stacklevel=2)
1247
+ self.resample_timestep(*args, **kwargs)
1248
+
1249
+ def getQuantilesFromKML(self, *args, **kwargs):
1250
+ warnings.warn('getQuantilesFromKML() is deprecated and will be removed in a future release; call get_quantiles_from_KML() directly instead.', DeprecationWarning, stacklevel=2)
1251
+ return self.get_quantiles_from_KML(*args, **kwargs)
1252
+
1253
+ def getDataFromLatLon(self, *args, **kwargs):
1254
+ warnings.warn('getDataFromLatLon() is deprecated and will be removed in a future release; call get_values_from_latlon() directly instead.', DeprecationWarning, stacklevel=2)
1255
+ return self.get_values_from_latlon(*args, **kwargs)
1256
+
1257
+ def getCropped(self, *args, **kwargs):
1258
+ warnings.warn('getCropped() is deprecated and will be removed in a future release; call get_cropped() directly instead.', DeprecationWarning, stacklevel=2)
1259
+ return self.get_cropped(*args, **kwargs)
1260
+
1261
+ def getValuesFromKML(self, *args, **kwargs):
1262
+ warnings.warn('getValuesFromKML() is deprecated and will be removed in a future release; call get_values_from_KML() directly instead.', DeprecationWarning, stacklevel=2)
1263
+ return self.get_values_from_KML(*args, **kwargs)
1264
+
1265
+ if __name__=='__main__':
1266
+
1267
+ pass