ginput 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. ginput/__init__.py +4 -0
  2. ginput/common_utils/__init__.py +0 -0
  3. ginput/common_utils/ggg_logging.py +62 -0
  4. ginput/common_utils/ioutils.py +304 -0
  5. ginput/common_utils/mod_constants.py +24 -0
  6. ginput/common_utils/mod_utils.py +2663 -0
  7. ginput/common_utils/readers.py +477 -0
  8. ginput/common_utils/run_utils.py +57 -0
  9. ginput/common_utils/sat_utils.py +67 -0
  10. ginput/common_utils/versioning.py +251 -0
  11. ginput/common_utils/writers.py +377 -0
  12. ginput/data/.gitignore +1 -0
  13. ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
  14. ginput/data/ML_monthly_obs_ch4.txt +498 -0
  15. ginput/data/ML_monthly_obs_co.txt +412 -0
  16. ginput/data/ML_monthly_obs_co2.txt +597 -0
  17. ginput/data/ML_monthly_obs_n2o.txt +330 -0
  18. ginput/data/SMO_monthly_obs_ch4.txt +499 -0
  19. ginput/data/SMO_monthly_obs_co.txt +416 -0
  20. ginput/data/SMO_monthly_obs_co2.txt +613 -0
  21. ginput/data/SMO_monthly_obs_n2o.txt +330 -0
  22. ginput/data/ace_fn2o_lut.nc +0 -0
  23. ginput/data/age_spectra/age.midlat.txt +47 -0
  24. ginput/data/age_spectra/age.tropics.txt +46 -0
  25. ginput/data/age_spectra/age.vortex.txt +31 -0
  26. ginput/data/age_spectra/agespec.midlat.txt +47 -0
  27. ginput/data/age_spectra/agespec.tropics.txt +46 -0
  28. ginput/data/age_spectra/agespec.vortex.txt +31 -0
  29. ginput/data/age_spectra/time.txt +3001 -0
  30. ginput/data/ch4_hf_slopes.nc +0 -0
  31. ginput/data/ch4_strat_lut.nc +0 -0
  32. ginput/data/clams_age_clim_scaled.nc +0 -0
  33. ginput/data/co2_strat_lut.nc +0 -0
  34. ginput/data/fo2_data/.gitignore +2 -0
  35. ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
  36. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
  37. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
  38. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
  39. ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
  40. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
  41. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
  42. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
  43. ginput/data/hf_strat_lut.nc +0 -0
  44. ginput/data/isotopologs.dat +199 -0
  45. ginput/data/meso_co_lut.nc +0 -0
  46. ginput/data/n2o_ch4_acefts.nc +0 -0
  47. ginput/data/n2o_strat_lut.nc +0 -0
  48. ginput/data/o2_mean_dmf.dat +52 -0
  49. ginput/data/summer_35N.vmr +182 -0
  50. ginput/data/washenfelder03_table_s3.txt +83 -0
  51. ginput/download/__init__.py +0 -0
  52. ginput/download/download_utils.py +46 -0
  53. ginput/download/get_GEOS5.py +367 -0
  54. ginput/download/get_MERRA2.py +177 -0
  55. ginput/download/get_NOAA_flask_data.py +103 -0
  56. ginput/download/get_fo2_data.py +140 -0
  57. ginput/mod_maker/__init__.py +0 -0
  58. ginput/mod_maker/mod_maker.py +2470 -0
  59. ginput/mod_maker/slantify.py +450 -0
  60. ginput/mod_maker/tccon_sites.py +570 -0
  61. ginput/priors/__init__.py +0 -0
  62. ginput/priors/acos_interface.py +1376 -0
  63. ginput/priors/automation.py +313 -0
  64. ginput/priors/backend_analysis/__init__.py +0 -0
  65. ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
  66. ginput/priors/backend_analysis/ace_fts_val.py +452 -0
  67. ginput/priors/backend_analysis/acos_comparison.py +550 -0
  68. ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
  69. ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
  70. ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
  71. ginput/priors/backend_analysis/backend_driver.py +90 -0
  72. ginput/priors/backend_analysis/backend_utils.py +358 -0
  73. ginput/priors/backend_analysis/clams.py +71 -0
  74. ginput/priors/backend_analysis/create_test_priors.py +853 -0
  75. ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
  76. ginput/priors/backend_analysis/mls_val.py +217 -0
  77. ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
  78. ginput/priors/fo2_prep.py +311 -0
  79. ginput/priors/map_maker.py +193 -0
  80. ginput/priors/mlo_smo_prep.py +1146 -0
  81. ginput/priors/tccon_priors.py +3482 -0
  82. ginput-1.3.0.dist-info/LICENSE.txt +203 -0
  83. ginput-1.3.0.dist-info/METADATA +125 -0
  84. ginput-1.3.0.dist-info/RECORD +86 -0
  85. ginput-1.3.0.dist-info/WHEEL +5 -0
  86. ginput-1.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,594 @@
1
+ """
2
+ This module contains the code necessary to regenerate the GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc file.
3
+
4
+ For the tropospheric prior profiles, we need to know the latitude of the observation in order to determine the age of
5
+ the air (to know how far forward or back to look in the MLO/SMO record) and the seasonal cycle (which is strongest in
6
+ the northern midlatitudes). During testing, we found that some sites should have more tropical CO2 profiles that their
7
+ geographic latitude suggests. That is, the aircraft or aircore CO2 profiles were relatively flat throughout the
8
+ troposphere (which is hallmark of the tropics due to rapid vertical mixing) but the priors were assigning them a more
9
+ midlatitude-like seasonal cycle.
10
+
11
+ The correction for this was to use an "equivalent latitude" based on the mid-tropospheric potential temperature. This
12
+ originates from work by Gretchen Keppel-Aleks (Keppel-Aleks et al., Biogeosciences, 9, 875-891,
13
+ https://doi.org/10.5194/bg-9-875-2012, 2012) that showed that CO2 in the free troposphere correlates much better with
14
+ potential temperature at 700 hPa than it does simple geographic latitude (c.f. especially figures 9-11).
15
+
16
+ In order to use this equivalent latitude based on potential temperature, we need a dataset that allows us to map the
17
+ typical mid-tropospheric potential temperature to the equivalent latitude it represents. To get that, we make biweekly
18
+ zonal bins of mid-tropospheric potential temperature vs. latitude in this module. The main driver function is
19
+ `make_geos_lat_v_theta_climatology`. To reproduce the standard 2018 file mentioned above, it would be called as::
20
+
21
+ make_geos_lat_v_theta_climatology(2018, geos_path, '.', target_pres=(500,700))
22
+
23
+ This would produce GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc in the current directory, which should then be placed
24
+ in the package :file:`data` directory. 2018 was chosen as the year for this climatology because it was the most recent
25
+ full year at the time of production, therefore it should be most representative looking backwards to the beginning of
26
+ the TCCON record and forward to the future TCCON record.
27
+
28
+ This method is necessary, rather than using potential vorticity based equivalent latitude, because Pan et al. 2012
29
+ (ACP, 12, p. 9187, doi: 10.5194/acp-12-9187-2012) note that when potential temperature surfaces intersect the surface,
30
+ potential temperature becomes a poor vertical coordinate. In general, they note that PV/EqLat can conceal tropospheric
31
+ chemical patterns.
32
+ """
33
+
34
+
35
+ import netCDF4 as ncdf
36
+ import numpy as np
37
+ import os
38
+ import pandas as pd
39
+
40
+ from ...common_utils.ioutils import make_ncdim_helper, make_ncvar_helper
41
+ from ...common_utils import mod_utils
42
+ from ...common_utils.mod_utils import ProgressBar
43
+
44
+ _mydir = os.path.dirname(__file__)
45
+ _theta_bin_edges = np.arange(220.0, 350.0, 2.0)
46
+ _theta_bin_centers = (_theta_bin_edges[:-1] + _theta_bin_edges[1:])/2.0
47
+
48
+
49
+ class GEOSError(Exception):
50
+ """
51
+ Base error for problems with the GEOS met data
52
+ """
53
+ pass
54
+
55
+
56
+ class GEOSDimensionError(GEOSError):
57
+ """
58
+ Error for cases where the dimension of a GEOS variable is not as expected
59
+ """
60
+ pass
61
+
62
+
63
+ class GEOSCoordinateError(GEOSError):
64
+ """
65
+ Error for any problem with the coordinates in the GEOS met file
66
+ """
67
+ pass
68
+
69
+
70
+ def bin_lat_vs_theta(lat, theta, pres_lev, target_pres=700, percentiles=np.arange(10.0, 100.0, 10.0), bin_var='latitude'):
71
+ """
72
+ Create zonal bins of latitude and theta and calculate statistics on them.
73
+
74
+ This is the main function that handles the calculation of the climatological latitude and potential temperature. It
75
+ can use either latitude or theta as the variable that defines the bins; by default latitude is used.
76
+
77
+ :param lat: a vector defining the latitude coordinates for the GEOS potential temperature fields. It will
78
+ automatically be expanded to the proper size to match ``theta``, and is expected to be the same length as the
79
+ second-to-last dimension of ``theta``.
80
+ :type lat: :class:`numpy.ndarray`
81
+
82
+ :param theta: an array defining the potential temperature from the GEOS met data.
83
+ :type theta: :class:`numpy.ndarray`
84
+
85
+ :param pres_lev: a vector defining the pressure levels as the vertical coordinate for ``theta``. It is expected to
86
+ be the same length as the third-from last dimension of ``theta``.
87
+ :type pres_lev: :class:`numpy.ndarray`
88
+
89
+ :param target_pres: the pressure or pressures to calculate theta on. If given as a single number, the level of
90
+ ``theta`` use for the theta bins will be the level closest to it. If given as a two-element collection (e.g.
91
+ ``[500, 700]``), then all layers in theta between those values (inclusive) will be averaged along the vertical
92
+ dimension before calculating the bins. In the second form, order does not matter (the lesser or greater number
93
+ may come first).
94
+ :type target_pres: int, float, or list(int or float)
95
+
96
+ :param percentiles: a vector defining the percentiles to calculate for each bin. Note that these are true
97
+ percentiles, i.e. `10` should be used, not `0.1`.
98
+ :type percentiles: array-like
99
+
100
+ :param bin_var: which variable to use to define the bins. May be ``'latitude'`` or ``'theta'``.
101
+ :type bin_var: str
102
+
103
+ :return: the bin centers, dictionaries for theta and lat containing the results of the binning as an array for each
104
+ statistic (mean, median, std, percentiles, and count), and the vector of percentiles used. This last value is
105
+ returned so that they can be written to the output file even if the default value was used. The statistics arrays
106
+ will be nbins-by-nvalues. For all stats except "percentiles", nvalues is 1 (there is only 1 value per bin). For
107
+ "percentiles", it will be equal to the number of percentiles requested.
108
+ :rtype: :class:`numpy.ndarray`, dict, dict, array-like
109
+ """
110
+ if np.ndim(theta) < 3 or np.ndim(theta) > 4:
111
+ raise ValueError('theta expected to be a 3- or 4- D array')
112
+ elif np.ndim(lat) != 1:
113
+ raise ValueError('lat expected to be a 1D array')
114
+ elif np.ndim(pres_lev) != 1:
115
+ raise ValueError('pres_lev expected to be a 1D array')
116
+
117
+ # If theta is 3D, make it 4D with only length 1 in the first dimension
118
+ if np.ndim(theta) == 3:
119
+ theta = theta.reshape((1,) + theta.shape)
120
+ elif theta.shape[1] != np.size(pres_lev) or theta.shape[2] != np.size(lat):
121
+ raise ValueError('theta is expected to have dimensions [ntimes by] nlev by nlat by nlon. The given input has '
122
+ 'shape {}'.format(theta.shape))
123
+
124
+ # This assumes that the same vector of pressures defines the levels for every column in the met data. This holds
125
+ # true for GEOS-FP data, where the values are on a fixed pressure grid, but will need revisited if other met data
126
+ # not on a fixed pressure grid is used.
127
+ if np.size(target_pres) == 1:
128
+ # levels always needs to be array-like so that we don't squeeze out the vertical dimension in theta when
129
+ # averaging
130
+ levels = [np.argmin(np.abs(pres_lev - target_pres))]
131
+ elif np.size(target_pres) == 2:
132
+ levels = (pres_lev >= np.min(target_pres)) & (pres_lev <= np.max(target_pres))
133
+ else:
134
+ raise TypeError('target_pres must either be a scalar value or a two-element collection')
135
+
136
+ n_times, n_lev, n_lat, n_lon = theta.shape
137
+
138
+ # Average theta over the levels requested. Make lat into an array the same shape as the post-averaging theta. Then
139
+ # flatten these arrays into vectors so that we can deal with them as a linear sequence of values.
140
+ lat_vec = np.zeros((n_times, n_lat, n_lon))
141
+ lat_vec[:] = lat.reshape((1, n_lat, 1))
142
+ theta_vec = np.mean(theta[:, levels, :, :], axis=1)
143
+
144
+ theta_vec = theta_vec.flatten()
145
+ lat_vec = lat_vec.flatten()
146
+
147
+ # Prepare for the binning
148
+ bin_ops = {'mean': np.nanmean, 'median': np.nanmedian, 'std': np.nanstd, 'count': np.size,
149
+ 'percentiles': lambda x: np.percentile(x, percentiles)}
150
+ # Figure out how large an array each operation returns - necessary to create the fill arrays if there's no data
151
+ # in a bin
152
+ dummy_arr = np.arange(100)
153
+ bin_ops_size = {name: np.size(op(dummy_arr)) for name, op in bin_ops.items()}
154
+
155
+ # Depending on which variables was told to be used for binning, get the correct bin centers and edges.
156
+ # digitize gives the indices of the upper edge of the bind, we want the bottom because that will work with the bin
157
+ # centers, hence the -1.
158
+ if bin_var == 'theta':
159
+ bin_centers = _theta_bin_centers
160
+ bin_edges = _theta_bin_edges
161
+ bin_inds = np.digitize(theta_vec, bin_edges) - 1
162
+ elif bin_var == 'latitude':
163
+ # Since the input latitude is always the same, we can calculate the bin edges from it
164
+ lat_half_res = np.mean(np.diff(lat))/2.0
165
+ bin_edges = np.concatenate((lat - lat_half_res, lat[-1:] + lat_half_res))
166
+ bin_centers = lat
167
+ bin_inds = np.digitize(lat_vec, bin_edges) - 1
168
+ else:
169
+ raise ValueError('bin_var "{}" is not recognized'.format(bin_var))
170
+
171
+ # Actually do the binning. It was easier conceptually to store the calculation of the stats for each bin as a list
172
+ # of dictionaries; later we rearrange these into one array per stat. This could probably be streamlined, but likely
173
+ # isn't worth doing so unless this part is unnecessarily slow.
174
+ lat_bin_stats = []
175
+ theta_bin_stats = []
176
+ for bin in range(np.size(bin_edges) - 1):
177
+ i_bin = bin_inds == bin
178
+ bin_lats = lat_vec[i_bin]
179
+ bin_thetas = theta_vec[i_bin]
180
+
181
+ lat_stat_dict = dict()
182
+ theta_stat_dict = dict()
183
+
184
+ for name, op in bin_ops.items():
185
+ # Some operations do not allow for an empty array, so deliberately set those to nans
186
+ if np.size(bin_lats) == 0:
187
+ n_values = bin_ops_size[name]
188
+ lat_stat_dict[name] = np.full((n_values,), np.nan)
189
+ theta_stat_dict[name] = np.full((n_values,), np.nan)
190
+ else:
191
+ lat_stat_dict[name] = op(bin_lats)
192
+ theta_stat_dict[name] = op(bin_thetas)
193
+
194
+ lat_bin_stats.append(lat_stat_dict)
195
+ theta_bin_stats.append(theta_stat_dict)
196
+
197
+ # Rearrange each stat into an array. First need to figure out the size of the array. For consistency, all will be
198
+ # 2D and be bins-by-values, so the means will be n-by-1 and the percentiles n-by-9 by default.
199
+ n_bins = np.size(theta_bin_stats)
200
+ theta_bin_out = dict()
201
+ lat_bin_out = dict()
202
+ for op in bin_ops:
203
+ # Get the result from the first bin to figure out the size in the 2nd dimension
204
+ n_values = bin_ops_size[op]
205
+ theta_op_results = np.full((n_bins, n_values), np.nan)
206
+ lat_op_results = np.full((n_bins, n_values), np.nan)
207
+ for i_bin in range(n_bins):
208
+ theta_op_results[i_bin, :] = theta_bin_stats[i_bin][op]
209
+ lat_op_results[i_bin, :] = lat_bin_stats[i_bin][op]
210
+
211
+ theta_bin_out[op] = theta_op_results
212
+ lat_bin_out[op] = lat_op_results
213
+
214
+ return bin_centers, theta_bin_out, lat_bin_out, percentiles
215
+
216
+
217
+ def load_geos_data(geos_path, start_date, end_date, hours=None, product='fpit', skip_if_missing=False):
218
+ """
219
+ Custom function for loading GEOS data specifically for the binning procedure.
220
+
221
+ :param geos_path: the path to where the GEOS FP-IT Np files are stored
222
+ :type geos_path: str
223
+
224
+ :param start_date: the first date to load GEOS data from
225
+ :type start_date: datetime-like
226
+
227
+ :param end_date: the last date (inclusive) to load GEOS data from
228
+ :type end_date: datetime-like
229
+
230
+ :param hours: which hours of the day (in UTC) to load GEOS data for. GEOS data is currently available at 0, 3, 6, 9,
231
+ 12, 15, 18, and 21 UTC. If this is ``None``, then all hours are loaded.
232
+ :type hours: None or list(int)
233
+
234
+ :param product: which GEOS product to load ('fp' or 'fpit')
235
+ :type product: str
236
+
237
+ :param skip_if_missing: set to ``True`` to allow there to be missing GEOS files in the date range. If there are,
238
+ then this will simply return three ``None`` values rather than raising an error (which is the default behavior).
239
+ Usually this is only set to ``True`` for testing this module with a subset of GEOS data.
240
+ :type skip_if_missing: bool
241
+
242
+ :return: latitude and pressure levels as vectors, theta as an array (ntimes by nlevels by nlat by nlon).
243
+ :rtype: :class:`numpy.ndarray` x 3 or None x 3
244
+ :raises GEOSError: if ``skip_if_missing`` is ``False`` and any expected GEOS files could not be found.
245
+
246
+ Note that this function should *only* be used to load data for this module. :func:`mod_utils.read_geos_files` should
247
+ be used elsewhere.
248
+ """
249
+ if hours is None:
250
+ # We always want hour to be iterable, so if it is a tuple with None in it, then we can iterate over it and
251
+ # when we pass None to geosfp_file_names, we'll get files between the start and end dates for all hours.
252
+ hours = (hours,)
253
+
254
+ print('Loading GEOS files for {} to {}'.format(start_date, end_date))
255
+ # Make the last day exclusive
256
+ end_date -= pd.Timedelta(days=1)
257
+ for hr in hours:
258
+ geos_names = mod_utils.geosfp_file_names_by_day(product, 'met', 'p', pd.date_range(start=start_date, end=end_date), utc_hours=hr)
259
+ pbar = ProgressBar(len(geos_names), prefix=' Progress:', add_one=True, style='counter')
260
+ for idx, fname in enumerate(geos_names):
261
+ full_name = os.path.join(geos_path, fname)
262
+ if not os.path.isfile(full_name):
263
+ if skip_if_missing:
264
+ return None, None, None
265
+ else:
266
+ raise GEOSError('Missing one of the files required for {} to {}'.format(start_date, end_date))
267
+ with ncdf.Dataset(full_name, 'r') as nci:
268
+ pbar.print_bar(idx)
269
+ # Need just latitude, theta and pressure levels. For the first file, just load theta (and make sure it's
270
+ # 4D). For the second file, append theta and verify that lat and pres levels are the same
271
+ this_lat = nci.variables['lat'][:]
272
+ this_plevs = nci.variables['lev'][:]
273
+ this_temperature = nci.variables['T'][:]
274
+ if np.ndim(this_temperature) == 3:
275
+ sz = (1,) + this_temperature.shape
276
+ this_temperature = this_temperature.reshape(sz)
277
+ elif np.ndim(this_temperature) != 4:
278
+ raise GEOSDimensionError('Temperature from GEOS file expected to be 3- or 4- D, not {}D'
279
+ .format(np.ndim(this_temperature)))
280
+
281
+ this_theta = mod_utils.calculate_model_potential_temperature(this_temperature, pres_levels=this_plevs)
282
+ this_theta = this_theta.filled(np.nan)
283
+
284
+ if idx == 0:
285
+ lat = this_lat
286
+ plevs = this_plevs
287
+ # initialize theta as ntimes by nlevels by nlat by nlon. The last three come from the shape of the
288
+ # currently read in theta - its first dimension is always 1 because there's 1 time per GEOS file.
289
+ theta = np.full((len(geos_names),) + this_theta.shape[1:], np.nan)
290
+ chk_file = full_name
291
+ else:
292
+ if not np.allclose(lat, this_lat):
293
+ raise GEOSCoordinateError('Latitudes from "{new_file}" differs from those in "{chk_file}"'
294
+ .format(new_file=full_name, chk_file=chk_file))
295
+ elif not np.allclose(plevs, this_plevs):
296
+ raise GEOSCoordinateError('Pressure levels from "{new_file}" differ from those in "{chk_file}"'
297
+ .format(new_file=full_name, chk_file=chk_file))
298
+
299
+ theta[idx] = this_theta
300
+
301
+ pbar.finish()
302
+
303
+ return lat, plevs, theta
304
+
305
+
306
+ def _cat_stats(stats):
307
+ """
308
+ Helper function to concatenate lists of statistics along a new first dimension.
309
+
310
+ :param stats: a list of arrays or list of dictionaries of arrays.
311
+ :return: a single array or dict of arrays with the original arrays from each list element concatenated along a new
312
+ first dimension.
313
+ """
314
+ def cat_stat_dict():
315
+ out_dict = dict()
316
+ stat_keys = stats[0].keys()
317
+ for k in stat_keys:
318
+ cat_arr = np.concatenate([s[k][np.newaxis] for s in stats], axis=0)
319
+ out_dict[k] = cat_arr
320
+
321
+ return out_dict
322
+
323
+ def cat_stat_simple():
324
+ return np.concatenate([s[np.newaxis] for s in stats], axis=0)
325
+
326
+ if isinstance(stats[0], dict):
327
+ return cat_stat_dict()
328
+ else:
329
+ return cat_stat_simple()
330
+
331
+
332
+ def _convert_dates(dates, base_date, calendar='gregorian'):
333
+ """
334
+ Helper function to convert dates into CF format.
335
+
336
+ :param dates: a collection of dates acceptable to :func:`netCDF4.date2num`
337
+ :param base_date: the date to use in the 'seconds since XX' unit
338
+ :param calendar: which CF calendar to use
339
+ :return: the array of dates as seconds since base_date, the units string, and the calendar name
340
+ """
341
+ units_str = 'seconds since {}'.format(base_date.strftime('%Y-%m-%d %H:%M:%S'))
342
+ date_arr = ncdf.date2num(dates, units_str, calendar=calendar)
343
+ return date_arr, units_str, calendar
344
+
345
+
346
+ def save_bin_ncdf_file(save_name, dates, hours, bin_var, bin_centers, theta_stats, lat_stats, percentiles, target_pres):
347
+ """
348
+ Save the netCDF file containing the theta and latitude bins.
349
+
350
+ :param save_name: the name to give the file. If exists, will be overwritten without warning
351
+ :type save_name: str
352
+
353
+ :param dates: a list of tuples, which each tuple is the start, midpoint, and end date for each averaging period. The
354
+ list must be the same length as the number of times in ``theta_stats`` and ``lat_stats``
355
+ :type dates: list(tuple(datetime, datetime, datetime))
356
+
357
+ :param hours: the vector of hours that corresponds to the second dimension of ``theta_stats`` and ``lat_stats``
358
+ arrays. If ``None``, then this dimension will be given a length of 1 and a value of NaN in the netCDF file.
359
+ :type hours: None or array-like
360
+
361
+ :param bin_var: which variable was used to set the bins. Will affect the name of the bin center variable in the
362
+ netCDF file.
363
+ :type bin_var: str
364
+
365
+ :param bin_centers: the vector of center points for the bins.
366
+ :type bin_centers: :class:`numpy.ndarray`
367
+
368
+ :param theta_stats: the dictionary of stats of theta. Each stat should be a key in the dictionary (e.g. 'mean',
369
+ 'std', etc.) and the value should be a ntimes x nhours x nbins x nvalues array. ntimes means the number of time
370
+ periods the year is separated into; nhours will be 1 unless the data were separated into different UTC hours.
371
+ :type theta_stats: dict
372
+
373
+ :param lat_stats: the dictionary of bin-by-bin stats for latitude. Same format as ``theta_stats``.
374
+ :type lat_stats: dict
375
+
376
+ :param percentiles: the array of percentiles use in the statistics.
377
+ :type percentiles: array-like
378
+
379
+ :param target_pres: the pressure or pressures theta was averaged on or between. Will be stored as an attribute in
380
+ the netCDF file.
381
+ :type target_pres: int, float, or list(int or float)
382
+
383
+ :return: None
384
+ """
385
+ def make_stat_dims(stat, std_dims, avail_dims):
386
+ dim_size_map = {d.size: d for d in avail_dims}
387
+ if len(dim_size_map) < len(avail_dims):
388
+ raise NotImplementedError('One or more dimensions have the same size')
389
+
390
+ dims = list(std_dims)
391
+ for i in range(len(std_dims), np.ndim(stat)):
392
+ dims.append(dim_size_map[stat.shape[i]])
393
+
394
+ return dims
395
+
396
+ if hours is None or hours == (None,):
397
+ hours = np.full((1,), np.nan)
398
+ else:
399
+ hours = np.array(hours)
400
+
401
+ # dates is a list of tuples of start, mid, and end dates. zip(*dates) will make an iterator that returns all start,
402
+ # then all mid, then all end dates in turn.
403
+ base_date = pd.Timestamp(1970, 1, 1)
404
+ start_dates, mid_dates, end_dates = [_convert_dates(d, base_date)[0] for d in zip(*dates)]
405
+
406
+ # convert one date to get the proper CF-style units string and calendar
407
+ _, date_unit_str, date_calendar = _convert_dates(dates[0][0], base_date)
408
+ common_date_attrs = {'units': date_unit_str, 'calendar': date_calendar}
409
+
410
+ with ncdf.Dataset(save_name, 'w') as nch:
411
+ # Record the target pressures as a root attribute
412
+ if isinstance(target_pres, (int, float)):
413
+ target_pres = (target_pres,)
414
+ nch.theta_range = '-'.join(str(p) for p in target_pres)
415
+
416
+ # First create the dimensions. Some don't have corresponding variables, so we don't use the helper function
417
+ # for those
418
+ times_attr = dict(description='Datetime at the middle of the binning period', **common_date_attrs)
419
+ times_dim = make_ncdim_helper(nch, 'times', mid_dates, **times_attr)
420
+
421
+ hours_dim = make_ncdim_helper(nch, 'hours', hours, units='UTC hour', description='UTC hour of the GEOS files used')
422
+ bins_dim = nch.createDimension('bins', bin_centers.shape[2])
423
+ percentiles_dim = make_ncdim_helper(nch, 'percentiles', percentiles)
424
+ values_dim = nch.createDimension('stat_values', 1)
425
+
426
+ std_3d_dims = (times_dim, hours_dim, bins_dim)
427
+ available_dims = [values_dim, percentiles_dim]
428
+
429
+ # Start writing the variables
430
+ make_ncvar_helper(nch, 'start_date', start_dates, (times_dim,), **common_date_attrs)
431
+ make_ncvar_helper(nch, 'end_date', end_dates, (times_dim,), **common_date_attrs)
432
+ if bin_var == 'theta':
433
+ bin_center_var_name = 'theta_bin_centers'
434
+ bin_center_description = 'Potential temperature at the center of the bin'
435
+ bin_center_units = 'K'
436
+ elif bin_var == 'latitude':
437
+ bin_center_var_name = 'latitude_bin_centers'
438
+ bin_center_description = 'Latitude at the center of the bin'
439
+ bin_center_units = 'degrees_north'
440
+
441
+ make_ncvar_helper(nch, bin_center_var_name, bin_centers, std_3d_dims,
442
+ units=bin_center_units, description=bin_center_description)
443
+
444
+ for var_name, var_unit, var in [('theta', 'K', theta_stats), ('latitude', 'degrees_north', lat_stats)]:
445
+ for stat_name, stat in var.items():
446
+ # For each of the stats variables, we need to add Nones to the dimension list to represent extra
447
+ # dimensions that need to be created by make_var_helper
448
+ stat_dims = make_stat_dims(stat, std_3d_dims, available_dims)
449
+ stat_nc_name = '{var}_{stat}'.format(var=var_name, stat=stat_name)
450
+ description = '{stat} of {var} in the bin'.format(stat=stat_name, var=var_name)
451
+
452
+ stat_attrs = {'units': var_unit, 'description': description}
453
+ if stat_name == 'count':
454
+ # Every stat should have the same units as the underlying quantity, except for count, which is just
455
+ # a number.
456
+ stat_attrs['units'] = 'number of data points'
457
+
458
+ make_ncvar_helper(nch, stat_nc_name, stat, stat_dims, **stat_attrs)
459
+
460
+
461
+ def iter_time_periods(year, freq):
462
+ """
463
+ Iterate over the averaging time periods
464
+ :param year: the year to use
465
+ :type year: datetime-like
466
+
467
+ :param freq: how long each time period is
468
+ :type freq: timedelta-like
469
+
470
+ :return: iterable over the time periods, yields the start, middle, and end dates of each time period.
471
+ """
472
+ def calc_end_date(sdate):
473
+ return sdate + freq
474
+
475
+ def calc_mid_date(sdate):
476
+ return sdate + freq / 2
477
+
478
+ start_date = pd.Timestamp(year, 1, 1)
479
+ end_date = calc_end_date(start_date)
480
+ mid_date = calc_mid_date(start_date)
481
+ # Yes, this will omit a day or two at the end of the year. But since 365 % 14 == 1, that means only Dec 31st won't
482
+ # get included in the climatology (or Dec 30-31st in a leap year). That is fine, because we're going to interpolate
483
+ # between the time periods anyway in the main code.
484
+ while end_date.year == year:
485
+ yield start_date, mid_date, end_date
486
+ start_date = end_date
487
+ end_date = calc_end_date(end_date)
488
+ mid_date = calc_mid_date(start_date)
489
+
490
+
491
+ def lat_v_theta_clim_name(year, target_pres, by_hour):
492
+ # Include the target pressure levels in the name to avoid accidentally overwriting files if we want to experiment
493
+ # with different pressure levels.
494
+ if isinstance(target_pres, (int, float)):
495
+ target_pres_name_str = '{}hPa'.format(target_pres)
496
+ else:
497
+ target_pres_name_str = '-'.join(str(p) for p in target_pres) + 'hPa'
498
+
499
+ if by_hour:
500
+ nc_file_name = 'GEOS_FPIT_lat_vs_theta_{}_{}_by_hour.nc'.format(year, target_pres_name_str)
501
+ else:
502
+ nc_file_name = 'GEOS_FPIT_lat_vs_theta_{}_{}.nc'.format(year, target_pres_name_str)
503
+
504
+ return nc_file_name
505
+
506
+
507
+ def make_geos_lat_v_theta_climatology(year, geos_path, save_path, freq=pd.Timedelta(days=14), by_hour=False,
508
+ skip_if_missing=False, bin_var='latitude', target_pres=700):
509
+ """
510
+ The main driver function to product the latitude vs. mid-tropospheric potential temperature climatology.
511
+
512
+ :param year: which year to use to generate the climatology
513
+ :type year: int
514
+
515
+ :param geos_path: where the GEOS FP or FP-IT Np (i.e. 3D pressure level files) are stored.
516
+ :type geos_path: str
517
+
518
+ :param save_path: where to save the resulting netCDF file
519
+ :type save_path: str
520
+
521
+ :param freq: how long each averaging time period should be
522
+ :type freq: timedelta-lik
523
+
524
+ :param by_hour: set to True to bin different UTC hours' GEOS files separately. This will make the "hours" dimension
525
+ in the netCDF file be > 1. This was an experimental feature and isn't used in the production climatology.
526
+ :type by_hour: bool
527
+
528
+ :param skip_if_missing: set to ``True`` to simply omit time periods missing one or more GEOS files from the
529
+ climatology rather than raising an error. This is intended for testing this code with a subset of GEOS data and
530
+ definitely should not be used in production.
531
+ :type skip_if_missing: bool
532
+
533
+ :param bin_var: which variable to use to define the bins. May be 'latitude' or 'theta'. Change this to 'theta' at
534
+ your own risk, as it will cause northern and southern latitudes to get binned together, which makes no sense.
535
+ Further, the main prior generating code expects the data to have been binned by latitude.
536
+ :type bin_var: str
537
+
538
+ :param target_pres: the pressure or pressures to calculate theta on. If given as a single number, the level of
539
+ ``theta`` use for the theta bins will be the level closest to it. If given as a two-element collection (e.g.
540
+ ``[500, 700]``), then all layers in theta between those values (inclusive) will be averaged along the vertical
541
+ dimension before calculating the bins. In the second form, order does not matter (the lesser or greater number
542
+ may come first).
543
+ :type target_pres: int, float, or list(int or float)
544
+
545
+ :return: None, saves netCDF file
546
+ """
547
+ if not by_hour:
548
+ hours = (None,)
549
+ else:
550
+ hours = list(range(0, 24, 3))
551
+
552
+ # The stats will be generated in several loops, one over time periods, and one over hours. As we finish each loop,
553
+ # the lists from that loop will get concatenated into single arrays with a new first dimension.
554
+ all_bin_centers = []
555
+ all_theta_stats = []
556
+ all_lat_stats = []
557
+ dates = []
558
+
559
+ for start, midpoint, end in iter_time_periods(year, freq):
560
+ bin_centers = []
561
+ theta_stats = []
562
+ lat_stats = []
563
+ do_save = True
564
+ for hr in hours:
565
+ # Usually this will just loop once, but it's in here as an experimental option to treat each UTC time
566
+ # separately. This never ended up being used, even in testing.
567
+ lat, plevs, theta = load_geos_data(geos_path, start, end, hours=(hr,), skip_if_missing=skip_if_missing)
568
+ if lat is None:
569
+ do_save = False
570
+ continue
571
+ this_bin_centers, this_theta_stats, this_lat_stats, percentiles = bin_lat_vs_theta(lat, theta, plevs,
572
+ bin_var=bin_var, target_pres=target_pres)
573
+ bin_centers.append(this_bin_centers)
574
+ theta_stats.append(this_theta_stats)
575
+ lat_stats.append(this_lat_stats)
576
+
577
+ if do_save:
578
+ # Concatenate the individual hours in the theta_stats, lat_stats, and bin_centers lists into single arrays
579
+ # that are (nhours)x(nbins)x(nvalues)
580
+ all_bin_centers.append(_cat_stats(bin_centers))
581
+ all_theta_stats.append(_cat_stats(theta_stats))
582
+ all_lat_stats.append(_cat_stats(lat_stats))
583
+ dates.append((start, midpoint, end))
584
+
585
+ # Convert to (ndays)x(nhours)x(nbins)x(nvalues)
586
+ all_bin_centers = _cat_stats(all_bin_centers)
587
+ all_theta_stats = _cat_stats(all_theta_stats)
588
+ all_lat_stats = _cat_stats(all_lat_stats)
589
+
590
+ nc_file_name = lat_v_theta_clim_name(year, target_pres, by_hour)
591
+ nc_file_name = os.path.join(save_path, nc_file_name)
592
+ save_bin_ncdf_file(nc_file_name, dates, hours, bin_var, all_bin_centers, all_theta_stats, all_lat_stats,
593
+ percentiles=percentiles, target_pres=target_pres)
594
+