ginput 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ginput/__init__.py +4 -0
- ginput/common_utils/__init__.py +0 -0
- ginput/common_utils/ggg_logging.py +62 -0
- ginput/common_utils/ioutils.py +304 -0
- ginput/common_utils/mod_constants.py +24 -0
- ginput/common_utils/mod_utils.py +2663 -0
- ginput/common_utils/readers.py +477 -0
- ginput/common_utils/run_utils.py +57 -0
- ginput/common_utils/sat_utils.py +67 -0
- ginput/common_utils/versioning.py +251 -0
- ginput/common_utils/writers.py +377 -0
- ginput/data/.gitignore +1 -0
- ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
- ginput/data/ML_monthly_obs_ch4.txt +498 -0
- ginput/data/ML_monthly_obs_co.txt +412 -0
- ginput/data/ML_monthly_obs_co2.txt +597 -0
- ginput/data/ML_monthly_obs_n2o.txt +330 -0
- ginput/data/SMO_monthly_obs_ch4.txt +499 -0
- ginput/data/SMO_monthly_obs_co.txt +416 -0
- ginput/data/SMO_monthly_obs_co2.txt +613 -0
- ginput/data/SMO_monthly_obs_n2o.txt +330 -0
- ginput/data/ace_fn2o_lut.nc +0 -0
- ginput/data/age_spectra/age.midlat.txt +47 -0
- ginput/data/age_spectra/age.tropics.txt +46 -0
- ginput/data/age_spectra/age.vortex.txt +31 -0
- ginput/data/age_spectra/agespec.midlat.txt +47 -0
- ginput/data/age_spectra/agespec.tropics.txt +46 -0
- ginput/data/age_spectra/agespec.vortex.txt +31 -0
- ginput/data/age_spectra/time.txt +3001 -0
- ginput/data/ch4_hf_slopes.nc +0 -0
- ginput/data/ch4_strat_lut.nc +0 -0
- ginput/data/clams_age_clim_scaled.nc +0 -0
- ginput/data/co2_strat_lut.nc +0 -0
- ginput/data/fo2_data/.gitignore +2 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
- ginput/data/hf_strat_lut.nc +0 -0
- ginput/data/isotopologs.dat +199 -0
- ginput/data/meso_co_lut.nc +0 -0
- ginput/data/n2o_ch4_acefts.nc +0 -0
- ginput/data/n2o_strat_lut.nc +0 -0
- ginput/data/o2_mean_dmf.dat +52 -0
- ginput/data/summer_35N.vmr +182 -0
- ginput/data/washenfelder03_table_s3.txt +83 -0
- ginput/download/__init__.py +0 -0
- ginput/download/download_utils.py +46 -0
- ginput/download/get_GEOS5.py +367 -0
- ginput/download/get_MERRA2.py +177 -0
- ginput/download/get_NOAA_flask_data.py +103 -0
- ginput/download/get_fo2_data.py +140 -0
- ginput/mod_maker/__init__.py +0 -0
- ginput/mod_maker/mod_maker.py +2470 -0
- ginput/mod_maker/slantify.py +450 -0
- ginput/mod_maker/tccon_sites.py +570 -0
- ginput/priors/__init__.py +0 -0
- ginput/priors/acos_interface.py +1376 -0
- ginput/priors/automation.py +313 -0
- ginput/priors/backend_analysis/__init__.py +0 -0
- ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
- ginput/priors/backend_analysis/ace_fts_val.py +452 -0
- ginput/priors/backend_analysis/acos_comparison.py +550 -0
- ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
- ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
- ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
- ginput/priors/backend_analysis/backend_driver.py +90 -0
- ginput/priors/backend_analysis/backend_utils.py +358 -0
- ginput/priors/backend_analysis/clams.py +71 -0
- ginput/priors/backend_analysis/create_test_priors.py +853 -0
- ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
- ginput/priors/backend_analysis/mls_val.py +217 -0
- ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
- ginput/priors/fo2_prep.py +311 -0
- ginput/priors/map_maker.py +193 -0
- ginput/priors/mlo_smo_prep.py +1146 -0
- ginput/priors/tccon_priors.py +3482 -0
- ginput-1.3.0.dist-info/LICENSE.txt +203 -0
- ginput-1.3.0.dist-info/METADATA +125 -0
- ginput-1.3.0.dist-info/RECORD +86 -0
- ginput-1.3.0.dist-info/WHEEL +5 -0
- ginput-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module contains the code necessary to regenerate the GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc file.
|
|
3
|
+
|
|
4
|
+
For the tropospheric prior profiles, we need to know the latitude of the observation in order to determine the age of
|
|
5
|
+
the air (to know how far forward or back to look in the MLO/SMO record) and the seasonal cycle (which is strongest in
|
|
6
|
+
the northern midlatitudes). During testing, we found that some sites should have more tropical CO2 profiles that their
|
|
7
|
+
geographic latitude suggests. That is, the aircraft or aircore CO2 profiles were relatively flat throughout the
|
|
8
|
+
troposphere (which is hallmark of the tropics due to rapid vertical mixing) but the priors were assigning them a more
|
|
9
|
+
midlatitude-like seasonal cycle.
|
|
10
|
+
|
|
11
|
+
The correction for this was to use an "equivalent latitude" based on the mid-tropospheric potential temperature. This
|
|
12
|
+
originates from work by Gretchen Keppel-Aleks (Keppel-Aleks et al., Biogeosciences, 9, 875-891,
|
|
13
|
+
https://doi.org/10.5194/bg-9-875-2012, 2012) that showed that CO2 in the free troposphere correlates much better with
|
|
14
|
+
potential temperature at 700 hPa than it does simple geographic latitude (c.f. especially figures 9-11).
|
|
15
|
+
|
|
16
|
+
In order to use this equivalent latitude based on potential temperature, we need a dataset that allows us to map the
|
|
17
|
+
typical mid-tropospheric potential temperature to the equivalent latitude it represents. To get that, we make biweekly
|
|
18
|
+
zonal bins of mid-tropospheric potential temperature vs. latitude in this module. The main driver function is
|
|
19
|
+
`make_geos_lat_v_theta_climatology`. To reproduce the standard 2018 file mentioned above, it would be called as::
|
|
20
|
+
|
|
21
|
+
make_geos_lat_v_theta_climatology(2018, geos_path, '.', target_pres=(500,700))
|
|
22
|
+
|
|
23
|
+
This would produce GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc in the current directory, which should then be placed
|
|
24
|
+
in the package :file:`data` directory. 2018 was chosen as the year for this climatology because it was the most recent
|
|
25
|
+
full year at the time of production, therefore it should be most representative looking backwards to the beginning of
|
|
26
|
+
the TCCON record and forward to the future TCCON record.
|
|
27
|
+
|
|
28
|
+
This method is necessary, rather than using potential vorticity based equivalent latitude, because Pan et al. 2012
|
|
29
|
+
(ACP, 12, p. 9187, doi: 10.5194/acp-12-9187-2012) note that when potential temperature surfaces intersect the surface,
|
|
30
|
+
potential temperature becomes a poor vertical coordinate. In general, they note that PV/EqLat can conceal tropospheric
|
|
31
|
+
chemical patterns.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
import netCDF4 as ncdf
|
|
36
|
+
import numpy as np
|
|
37
|
+
import os
|
|
38
|
+
import pandas as pd
|
|
39
|
+
|
|
40
|
+
from ...common_utils.ioutils import make_ncdim_helper, make_ncvar_helper
|
|
41
|
+
from ...common_utils import mod_utils
|
|
42
|
+
from ...common_utils.mod_utils import ProgressBar
|
|
43
|
+
|
|
44
|
+
_mydir = os.path.dirname(__file__)
|
|
45
|
+
_theta_bin_edges = np.arange(220.0, 350.0, 2.0)
|
|
46
|
+
_theta_bin_centers = (_theta_bin_edges[:-1] + _theta_bin_edges[1:])/2.0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class GEOSError(Exception):
|
|
50
|
+
"""
|
|
51
|
+
Base error for problems with the GEOS met data
|
|
52
|
+
"""
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class GEOSDimensionError(GEOSError):
|
|
57
|
+
"""
|
|
58
|
+
Error for cases where the dimension of a GEOS variable is not as expected
|
|
59
|
+
"""
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class GEOSCoordinateError(GEOSError):
|
|
64
|
+
"""
|
|
65
|
+
Error for any problem with the coordinates in the GEOS met file
|
|
66
|
+
"""
|
|
67
|
+
pass
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def bin_lat_vs_theta(lat, theta, pres_lev, target_pres=700, percentiles=np.arange(10.0, 100.0, 10.0), bin_var='latitude'):
|
|
71
|
+
"""
|
|
72
|
+
Create zonal bins of latitude and theta and calculate statistics on them.
|
|
73
|
+
|
|
74
|
+
This is the main function that handles the calculation of the climatological latitude and potential temperature. It
|
|
75
|
+
can use either latitude or theta as the variable that defines the bins; by default latitude is used.
|
|
76
|
+
|
|
77
|
+
:param lat: a vector defining the latitude coordinates for the GEOS potential temperature fields. It will
|
|
78
|
+
automatically be expanded to the proper size to match ``theta``, and is expected to be the same length as the
|
|
79
|
+
second-to-last dimension of ``theta``.
|
|
80
|
+
:type lat: :class:`numpy.ndarray`
|
|
81
|
+
|
|
82
|
+
:param theta: an array defining the potential temperature from the GEOS met data.
|
|
83
|
+
:type theta: :class:`numpy.ndarray`
|
|
84
|
+
|
|
85
|
+
:param pres_lev: a vector defining the pressure levels as the vertical coordinate for ``theta``. It is expected to
|
|
86
|
+
be the same length as the third-from last dimension of ``theta``.
|
|
87
|
+
:type pres_lev: :class:`numpy.ndarray`
|
|
88
|
+
|
|
89
|
+
:param target_pres: the pressure or pressures to calculate theta on. If given as a single number, the level of
|
|
90
|
+
``theta`` use for the theta bins will be the level closest to it. If given as a two-element collection (e.g.
|
|
91
|
+
``[500, 700]``), then all layers in theta between those values (inclusive) will be averaged along the vertical
|
|
92
|
+
dimension before calculating the bins. In the second form, order does not matter (the lesser or greater number
|
|
93
|
+
may come first).
|
|
94
|
+
:type target_pres: int, float, or list(int or float)
|
|
95
|
+
|
|
96
|
+
:param percentiles: a vector defining the percentiles to calculate for each bin. Note that these are true
|
|
97
|
+
percentiles, i.e. `10` should be used, not `0.1`.
|
|
98
|
+
:type percentiles: array-like
|
|
99
|
+
|
|
100
|
+
:param bin_var: which variable to use to define the bins. May be ``'latitude'`` or ``'theta'``.
|
|
101
|
+
:type bin_var: str
|
|
102
|
+
|
|
103
|
+
:return: the bin centers, dictionaries for theta and lat containing the results of the binning as an array for each
|
|
104
|
+
statistic (mean, median, std, percentiles, and count), and the vector of percentiles used. This last value is
|
|
105
|
+
returned so that they can be written to the output file even if the default value was used. The statistics arrays
|
|
106
|
+
will be nbins-by-nvalues. For all stats except "percentiles", nvalues is 1 (there is only 1 value per bin). For
|
|
107
|
+
"percentiles", it will be equal to the number of percentiles requested.
|
|
108
|
+
:rtype: :class:`numpy.ndarray`, dict, dict, array-like
|
|
109
|
+
"""
|
|
110
|
+
if np.ndim(theta) < 3 or np.ndim(theta) > 4:
|
|
111
|
+
raise ValueError('theta expected to be a 3- or 4- D array')
|
|
112
|
+
elif np.ndim(lat) != 1:
|
|
113
|
+
raise ValueError('lat expected to be a 1D array')
|
|
114
|
+
elif np.ndim(pres_lev) != 1:
|
|
115
|
+
raise ValueError('pres_lev expected to be a 1D array')
|
|
116
|
+
|
|
117
|
+
# If theta is 3D, make it 4D with only length 1 in the first dimension
|
|
118
|
+
if np.ndim(theta) == 3:
|
|
119
|
+
theta = theta.reshape((1,) + theta.shape)
|
|
120
|
+
elif theta.shape[1] != np.size(pres_lev) or theta.shape[2] != np.size(lat):
|
|
121
|
+
raise ValueError('theta is expected to have dimensions [ntimes by] nlev by nlat by nlon. The given input has '
|
|
122
|
+
'shape {}'.format(theta.shape))
|
|
123
|
+
|
|
124
|
+
# This assumes that the same vector of pressures defines the levels for every column in the met data. This holds
|
|
125
|
+
# true for GEOS-FP data, where the values are on a fixed pressure grid, but will need revisited if other met data
|
|
126
|
+
# not on a fixed pressure grid is used.
|
|
127
|
+
if np.size(target_pres) == 1:
|
|
128
|
+
# levels always needs to be array-like so that we don't squeeze out the vertical dimension in theta when
|
|
129
|
+
# averaging
|
|
130
|
+
levels = [np.argmin(np.abs(pres_lev - target_pres))]
|
|
131
|
+
elif np.size(target_pres) == 2:
|
|
132
|
+
levels = (pres_lev >= np.min(target_pres)) & (pres_lev <= np.max(target_pres))
|
|
133
|
+
else:
|
|
134
|
+
raise TypeError('target_pres must either be a scalar value or a two-element collection')
|
|
135
|
+
|
|
136
|
+
n_times, n_lev, n_lat, n_lon = theta.shape
|
|
137
|
+
|
|
138
|
+
# Average theta over the levels requested. Make lat into an array the same shape as the post-averaging theta. Then
|
|
139
|
+
# flatten these arrays into vectors so that we can deal with them as a linear sequence of values.
|
|
140
|
+
lat_vec = np.zeros((n_times, n_lat, n_lon))
|
|
141
|
+
lat_vec[:] = lat.reshape((1, n_lat, 1))
|
|
142
|
+
theta_vec = np.mean(theta[:, levels, :, :], axis=1)
|
|
143
|
+
|
|
144
|
+
theta_vec = theta_vec.flatten()
|
|
145
|
+
lat_vec = lat_vec.flatten()
|
|
146
|
+
|
|
147
|
+
# Prepare for the binning
|
|
148
|
+
bin_ops = {'mean': np.nanmean, 'median': np.nanmedian, 'std': np.nanstd, 'count': np.size,
|
|
149
|
+
'percentiles': lambda x: np.percentile(x, percentiles)}
|
|
150
|
+
# Figure out how large an array each operation returns - necessary to create the fill arrays if there's no data
|
|
151
|
+
# in a bin
|
|
152
|
+
dummy_arr = np.arange(100)
|
|
153
|
+
bin_ops_size = {name: np.size(op(dummy_arr)) for name, op in bin_ops.items()}
|
|
154
|
+
|
|
155
|
+
# Depending on which variables was told to be used for binning, get the correct bin centers and edges.
|
|
156
|
+
# digitize gives the indices of the upper edge of the bind, we want the bottom because that will work with the bin
|
|
157
|
+
# centers, hence the -1.
|
|
158
|
+
if bin_var == 'theta':
|
|
159
|
+
bin_centers = _theta_bin_centers
|
|
160
|
+
bin_edges = _theta_bin_edges
|
|
161
|
+
bin_inds = np.digitize(theta_vec, bin_edges) - 1
|
|
162
|
+
elif bin_var == 'latitude':
|
|
163
|
+
# Since the input latitude is always the same, we can calculate the bin edges from it
|
|
164
|
+
lat_half_res = np.mean(np.diff(lat))/2.0
|
|
165
|
+
bin_edges = np.concatenate((lat - lat_half_res, lat[-1:] + lat_half_res))
|
|
166
|
+
bin_centers = lat
|
|
167
|
+
bin_inds = np.digitize(lat_vec, bin_edges) - 1
|
|
168
|
+
else:
|
|
169
|
+
raise ValueError('bin_var "{}" is not recognized'.format(bin_var))
|
|
170
|
+
|
|
171
|
+
# Actually do the binning. It was easier conceptually to store the calculation of the stats for each bin as a list
|
|
172
|
+
# of dictionaries; later we rearrange these into one array per stat. This could probably be streamlined, but likely
|
|
173
|
+
# isn't worth doing so unless this part is unnecessarily slow.
|
|
174
|
+
lat_bin_stats = []
|
|
175
|
+
theta_bin_stats = []
|
|
176
|
+
for bin in range(np.size(bin_edges) - 1):
|
|
177
|
+
i_bin = bin_inds == bin
|
|
178
|
+
bin_lats = lat_vec[i_bin]
|
|
179
|
+
bin_thetas = theta_vec[i_bin]
|
|
180
|
+
|
|
181
|
+
lat_stat_dict = dict()
|
|
182
|
+
theta_stat_dict = dict()
|
|
183
|
+
|
|
184
|
+
for name, op in bin_ops.items():
|
|
185
|
+
# Some operations do not allow for an empty array, so deliberately set those to nans
|
|
186
|
+
if np.size(bin_lats) == 0:
|
|
187
|
+
n_values = bin_ops_size[name]
|
|
188
|
+
lat_stat_dict[name] = np.full((n_values,), np.nan)
|
|
189
|
+
theta_stat_dict[name] = np.full((n_values,), np.nan)
|
|
190
|
+
else:
|
|
191
|
+
lat_stat_dict[name] = op(bin_lats)
|
|
192
|
+
theta_stat_dict[name] = op(bin_thetas)
|
|
193
|
+
|
|
194
|
+
lat_bin_stats.append(lat_stat_dict)
|
|
195
|
+
theta_bin_stats.append(theta_stat_dict)
|
|
196
|
+
|
|
197
|
+
# Rearrange each stat into an array. First need to figure out the size of the array. For consistency, all will be
|
|
198
|
+
# 2D and be bins-by-values, so the means will be n-by-1 and the percentiles n-by-9 by default.
|
|
199
|
+
n_bins = np.size(theta_bin_stats)
|
|
200
|
+
theta_bin_out = dict()
|
|
201
|
+
lat_bin_out = dict()
|
|
202
|
+
for op in bin_ops:
|
|
203
|
+
# Get the result from the first bin to figure out the size in the 2nd dimension
|
|
204
|
+
n_values = bin_ops_size[op]
|
|
205
|
+
theta_op_results = np.full((n_bins, n_values), np.nan)
|
|
206
|
+
lat_op_results = np.full((n_bins, n_values), np.nan)
|
|
207
|
+
for i_bin in range(n_bins):
|
|
208
|
+
theta_op_results[i_bin, :] = theta_bin_stats[i_bin][op]
|
|
209
|
+
lat_op_results[i_bin, :] = lat_bin_stats[i_bin][op]
|
|
210
|
+
|
|
211
|
+
theta_bin_out[op] = theta_op_results
|
|
212
|
+
lat_bin_out[op] = lat_op_results
|
|
213
|
+
|
|
214
|
+
return bin_centers, theta_bin_out, lat_bin_out, percentiles
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def load_geos_data(geos_path, start_date, end_date, hours=None, product='fpit', skip_if_missing=False):
|
|
218
|
+
"""
|
|
219
|
+
Custom function for loading GEOS data specifically for the binning procedure.
|
|
220
|
+
|
|
221
|
+
:param geos_path: the path to where the GEOS FP-IT Np files are stored
|
|
222
|
+
:type geos_path: str
|
|
223
|
+
|
|
224
|
+
:param start_date: the first date to load GEOS data from
|
|
225
|
+
:type start_date: datetime-like
|
|
226
|
+
|
|
227
|
+
:param end_date: the last date (inclusive) to load GEOS data from
|
|
228
|
+
:type end_date: datetime-like
|
|
229
|
+
|
|
230
|
+
:param hours: which hours of the day (in UTC) to load GEOS data for. GEOS data is currently available at 0, 3, 6, 9,
|
|
231
|
+
12, 15, 18, and 21 UTC. If this is ``None``, then all hours are loaded.
|
|
232
|
+
:type hours: None or list(int)
|
|
233
|
+
|
|
234
|
+
:param product: which GEOS product to load ('fp' or 'fpit')
|
|
235
|
+
:type product: str
|
|
236
|
+
|
|
237
|
+
:param skip_if_missing: set to ``True`` to allow there to be missing GEOS files in the date range. If there are,
|
|
238
|
+
then this will simply return three ``None`` values rather than raising an error (which is the default behavior).
|
|
239
|
+
Usually this is only set to ``True`` for testing this module with a subset of GEOS data.
|
|
240
|
+
:type skip_if_missing: bool
|
|
241
|
+
|
|
242
|
+
:return: latitude and pressure levels as vectors, theta as an array (ntimes by nlevels by nlat by nlon).
|
|
243
|
+
:rtype: :class:`numpy.ndarray` x 3 or None x 3
|
|
244
|
+
:raises GEOSError: if ``skip_if_missing`` is ``False`` and any expected GEOS files could not be found.
|
|
245
|
+
|
|
246
|
+
Note that this function should *only* be used to load data for this module. :func:`mod_utils.read_geos_files` should
|
|
247
|
+
be used elsewhere.
|
|
248
|
+
"""
|
|
249
|
+
if hours is None:
|
|
250
|
+
# We always want hour to be iterable, so if it is a tuple with None in it, then we can iterate over it and
|
|
251
|
+
# when we pass None to geosfp_file_names, we'll get files between the start and end dates for all hours.
|
|
252
|
+
hours = (hours,)
|
|
253
|
+
|
|
254
|
+
print('Loading GEOS files for {} to {}'.format(start_date, end_date))
|
|
255
|
+
# Make the last day exclusive
|
|
256
|
+
end_date -= pd.Timedelta(days=1)
|
|
257
|
+
for hr in hours:
|
|
258
|
+
geos_names = mod_utils.geosfp_file_names_by_day(product, 'met', 'p', pd.date_range(start=start_date, end=end_date), utc_hours=hr)
|
|
259
|
+
pbar = ProgressBar(len(geos_names), prefix=' Progress:', add_one=True, style='counter')
|
|
260
|
+
for idx, fname in enumerate(geos_names):
|
|
261
|
+
full_name = os.path.join(geos_path, fname)
|
|
262
|
+
if not os.path.isfile(full_name):
|
|
263
|
+
if skip_if_missing:
|
|
264
|
+
return None, None, None
|
|
265
|
+
else:
|
|
266
|
+
raise GEOSError('Missing one of the files required for {} to {}'.format(start_date, end_date))
|
|
267
|
+
with ncdf.Dataset(full_name, 'r') as nci:
|
|
268
|
+
pbar.print_bar(idx)
|
|
269
|
+
# Need just latitude, theta and pressure levels. For the first file, just load theta (and make sure it's
|
|
270
|
+
# 4D). For the second file, append theta and verify that lat and pres levels are the same
|
|
271
|
+
this_lat = nci.variables['lat'][:]
|
|
272
|
+
this_plevs = nci.variables['lev'][:]
|
|
273
|
+
this_temperature = nci.variables['T'][:]
|
|
274
|
+
if np.ndim(this_temperature) == 3:
|
|
275
|
+
sz = (1,) + this_temperature.shape
|
|
276
|
+
this_temperature = this_temperature.reshape(sz)
|
|
277
|
+
elif np.ndim(this_temperature) != 4:
|
|
278
|
+
raise GEOSDimensionError('Temperature from GEOS file expected to be 3- or 4- D, not {}D'
|
|
279
|
+
.format(np.ndim(this_temperature)))
|
|
280
|
+
|
|
281
|
+
this_theta = mod_utils.calculate_model_potential_temperature(this_temperature, pres_levels=this_plevs)
|
|
282
|
+
this_theta = this_theta.filled(np.nan)
|
|
283
|
+
|
|
284
|
+
if idx == 0:
|
|
285
|
+
lat = this_lat
|
|
286
|
+
plevs = this_plevs
|
|
287
|
+
# initialize theta as ntimes by nlevels by nlat by nlon. The last three come from the shape of the
|
|
288
|
+
# currently read in theta - its first dimension is always 1 because there's 1 time per GEOS file.
|
|
289
|
+
theta = np.full((len(geos_names),) + this_theta.shape[1:], np.nan)
|
|
290
|
+
chk_file = full_name
|
|
291
|
+
else:
|
|
292
|
+
if not np.allclose(lat, this_lat):
|
|
293
|
+
raise GEOSCoordinateError('Latitudes from "{new_file}" differs from those in "{chk_file}"'
|
|
294
|
+
.format(new_file=full_name, chk_file=chk_file))
|
|
295
|
+
elif not np.allclose(plevs, this_plevs):
|
|
296
|
+
raise GEOSCoordinateError('Pressure levels from "{new_file}" differ from those in "{chk_file}"'
|
|
297
|
+
.format(new_file=full_name, chk_file=chk_file))
|
|
298
|
+
|
|
299
|
+
theta[idx] = this_theta
|
|
300
|
+
|
|
301
|
+
pbar.finish()
|
|
302
|
+
|
|
303
|
+
return lat, plevs, theta
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _cat_stats(stats):
|
|
307
|
+
"""
|
|
308
|
+
Helper function to concatenate lists of statistics along a new first dimension.
|
|
309
|
+
|
|
310
|
+
:param stats: a list of arrays or list of dictionaries of arrays.
|
|
311
|
+
:return: a single array or dict of arrays with the original arrays from each list element concatenated along a new
|
|
312
|
+
first dimension.
|
|
313
|
+
"""
|
|
314
|
+
def cat_stat_dict():
|
|
315
|
+
out_dict = dict()
|
|
316
|
+
stat_keys = stats[0].keys()
|
|
317
|
+
for k in stat_keys:
|
|
318
|
+
cat_arr = np.concatenate([s[k][np.newaxis] for s in stats], axis=0)
|
|
319
|
+
out_dict[k] = cat_arr
|
|
320
|
+
|
|
321
|
+
return out_dict
|
|
322
|
+
|
|
323
|
+
def cat_stat_simple():
|
|
324
|
+
return np.concatenate([s[np.newaxis] for s in stats], axis=0)
|
|
325
|
+
|
|
326
|
+
if isinstance(stats[0], dict):
|
|
327
|
+
return cat_stat_dict()
|
|
328
|
+
else:
|
|
329
|
+
return cat_stat_simple()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _convert_dates(dates, base_date, calendar='gregorian'):
|
|
333
|
+
"""
|
|
334
|
+
Helper function to convert dates into CF format.
|
|
335
|
+
|
|
336
|
+
:param dates: a collection of dates acceptable to :func:`netCDF4.date2num`
|
|
337
|
+
:param base_date: the date to use in the 'seconds since XX' unit
|
|
338
|
+
:param calendar: which CF calendar to use
|
|
339
|
+
:return: the array of dates as seconds since base_date, the units string, and the calendar name
|
|
340
|
+
"""
|
|
341
|
+
units_str = 'seconds since {}'.format(base_date.strftime('%Y-%m-%d %H:%M:%S'))
|
|
342
|
+
date_arr = ncdf.date2num(dates, units_str, calendar=calendar)
|
|
343
|
+
return date_arr, units_str, calendar
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def save_bin_ncdf_file(save_name, dates, hours, bin_var, bin_centers, theta_stats, lat_stats, percentiles, target_pres):
|
|
347
|
+
"""
|
|
348
|
+
Save the netCDF file containing the theta and latitude bins.
|
|
349
|
+
|
|
350
|
+
:param save_name: the name to give the file. If exists, will be overwritten without warning
|
|
351
|
+
:type save_name: str
|
|
352
|
+
|
|
353
|
+
:param dates: a list of tuples, which each tuple is the start, midpoint, and end date for each averaging period. The
|
|
354
|
+
list must be the same length as the number of times in ``theta_stats`` and ``lat_stats``
|
|
355
|
+
:type dates: list(tuple(datetime, datetime, datetime))
|
|
356
|
+
|
|
357
|
+
:param hours: the vector of hours that corresponds to the second dimension of ``theta_stats`` and ``lat_stats``
|
|
358
|
+
arrays. If ``None``, then this dimension will be given a length of 1 and a value of NaN in the netCDF file.
|
|
359
|
+
:type hours: None or array-like
|
|
360
|
+
|
|
361
|
+
:param bin_var: which variable was used to set the bins. Will affect the name of the bin center variable in the
|
|
362
|
+
netCDF file.
|
|
363
|
+
:type bin_var: str
|
|
364
|
+
|
|
365
|
+
:param bin_centers: the vector of center points for the bins.
|
|
366
|
+
:type bin_centers: :class:`numpy.ndarray`
|
|
367
|
+
|
|
368
|
+
:param theta_stats: the dictionary of stats of theta. Each stat should be a key in the dictionary (e.g. 'mean',
|
|
369
|
+
'std', etc.) and the value should be a ntimes x nhours x nbins x nvalues array. ntimes means the number of time
|
|
370
|
+
periods the year is separated into; nhours will be 1 unless the data were separated into different UTC hours.
|
|
371
|
+
:type theta_stats: dict
|
|
372
|
+
|
|
373
|
+
:param lat_stats: the dictionary of bin-by-bin stats for latitude. Same format as ``theta_stats``.
|
|
374
|
+
:type lat_stats: dict
|
|
375
|
+
|
|
376
|
+
:param percentiles: the array of percentiles use in the statistics.
|
|
377
|
+
:type percentiles: array-like
|
|
378
|
+
|
|
379
|
+
:param target_pres: the pressure or pressures theta was averaged on or between. Will be stored as an attribute in
|
|
380
|
+
the netCDF file.
|
|
381
|
+
:type target_pres: int, float, or list(int or float)
|
|
382
|
+
|
|
383
|
+
:return: None
|
|
384
|
+
"""
|
|
385
|
+
def make_stat_dims(stat, std_dims, avail_dims):
|
|
386
|
+
dim_size_map = {d.size: d for d in avail_dims}
|
|
387
|
+
if len(dim_size_map) < len(avail_dims):
|
|
388
|
+
raise NotImplementedError('One or more dimensions have the same size')
|
|
389
|
+
|
|
390
|
+
dims = list(std_dims)
|
|
391
|
+
for i in range(len(std_dims), np.ndim(stat)):
|
|
392
|
+
dims.append(dim_size_map[stat.shape[i]])
|
|
393
|
+
|
|
394
|
+
return dims
|
|
395
|
+
|
|
396
|
+
if hours is None or hours == (None,):
|
|
397
|
+
hours = np.full((1,), np.nan)
|
|
398
|
+
else:
|
|
399
|
+
hours = np.array(hours)
|
|
400
|
+
|
|
401
|
+
# dates is a list of tuples of start, mid, and end dates. zip(*dates) will make an iterator that returns all start,
|
|
402
|
+
# then all mid, then all end dates in turn.
|
|
403
|
+
base_date = pd.Timestamp(1970, 1, 1)
|
|
404
|
+
start_dates, mid_dates, end_dates = [_convert_dates(d, base_date)[0] for d in zip(*dates)]
|
|
405
|
+
|
|
406
|
+
# convert one date to get the proper CF-style units string and calendar
|
|
407
|
+
_, date_unit_str, date_calendar = _convert_dates(dates[0][0], base_date)
|
|
408
|
+
common_date_attrs = {'units': date_unit_str, 'calendar': date_calendar}
|
|
409
|
+
|
|
410
|
+
with ncdf.Dataset(save_name, 'w') as nch:
|
|
411
|
+
# Record the target pressures as a root attribute
|
|
412
|
+
if isinstance(target_pres, (int, float)):
|
|
413
|
+
target_pres = (target_pres,)
|
|
414
|
+
nch.theta_range = '-'.join(str(p) for p in target_pres)
|
|
415
|
+
|
|
416
|
+
# First create the dimensions. Some don't have corresponding variables, so we don't use the helper function
|
|
417
|
+
# for those
|
|
418
|
+
times_attr = dict(description='Datetime at the middle of the binning period', **common_date_attrs)
|
|
419
|
+
times_dim = make_ncdim_helper(nch, 'times', mid_dates, **times_attr)
|
|
420
|
+
|
|
421
|
+
hours_dim = make_ncdim_helper(nch, 'hours', hours, units='UTC hour', description='UTC hour of the GEOS files used')
|
|
422
|
+
bins_dim = nch.createDimension('bins', bin_centers.shape[2])
|
|
423
|
+
percentiles_dim = make_ncdim_helper(nch, 'percentiles', percentiles)
|
|
424
|
+
values_dim = nch.createDimension('stat_values', 1)
|
|
425
|
+
|
|
426
|
+
std_3d_dims = (times_dim, hours_dim, bins_dim)
|
|
427
|
+
available_dims = [values_dim, percentiles_dim]
|
|
428
|
+
|
|
429
|
+
# Start writing the variables
|
|
430
|
+
make_ncvar_helper(nch, 'start_date', start_dates, (times_dim,), **common_date_attrs)
|
|
431
|
+
make_ncvar_helper(nch, 'end_date', end_dates, (times_dim,), **common_date_attrs)
|
|
432
|
+
if bin_var == 'theta':
|
|
433
|
+
bin_center_var_name = 'theta_bin_centers'
|
|
434
|
+
bin_center_description = 'Potential temperature at the center of the bin'
|
|
435
|
+
bin_center_units = 'K'
|
|
436
|
+
elif bin_var == 'latitude':
|
|
437
|
+
bin_center_var_name = 'latitude_bin_centers'
|
|
438
|
+
bin_center_description = 'Latitude at the center of the bin'
|
|
439
|
+
bin_center_units = 'degrees_north'
|
|
440
|
+
|
|
441
|
+
make_ncvar_helper(nch, bin_center_var_name, bin_centers, std_3d_dims,
|
|
442
|
+
units=bin_center_units, description=bin_center_description)
|
|
443
|
+
|
|
444
|
+
for var_name, var_unit, var in [('theta', 'K', theta_stats), ('latitude', 'degrees_north', lat_stats)]:
|
|
445
|
+
for stat_name, stat in var.items():
|
|
446
|
+
# For each of the stats variables, we need to add Nones to the dimension list to represent extra
|
|
447
|
+
# dimensions that need to be created by make_var_helper
|
|
448
|
+
stat_dims = make_stat_dims(stat, std_3d_dims, available_dims)
|
|
449
|
+
stat_nc_name = '{var}_{stat}'.format(var=var_name, stat=stat_name)
|
|
450
|
+
description = '{stat} of {var} in the bin'.format(stat=stat_name, var=var_name)
|
|
451
|
+
|
|
452
|
+
stat_attrs = {'units': var_unit, 'description': description}
|
|
453
|
+
if stat_name == 'count':
|
|
454
|
+
# Every stat should have the same units as the underlying quantity, except for count, which is just
|
|
455
|
+
# a number.
|
|
456
|
+
stat_attrs['units'] = 'number of data points'
|
|
457
|
+
|
|
458
|
+
make_ncvar_helper(nch, stat_nc_name, stat, stat_dims, **stat_attrs)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def iter_time_periods(year, freq):
|
|
462
|
+
"""
|
|
463
|
+
Iterate over the averaging time periods
|
|
464
|
+
:param year: the year to use
|
|
465
|
+
:type year: datetime-like
|
|
466
|
+
|
|
467
|
+
:param freq: how long each time period is
|
|
468
|
+
:type freq: timedelta-like
|
|
469
|
+
|
|
470
|
+
:return: iterable over the time periods, yields the start, middle, and end dates of each time period.
|
|
471
|
+
"""
|
|
472
|
+
def calc_end_date(sdate):
|
|
473
|
+
return sdate + freq
|
|
474
|
+
|
|
475
|
+
def calc_mid_date(sdate):
|
|
476
|
+
return sdate + freq / 2
|
|
477
|
+
|
|
478
|
+
start_date = pd.Timestamp(year, 1, 1)
|
|
479
|
+
end_date = calc_end_date(start_date)
|
|
480
|
+
mid_date = calc_mid_date(start_date)
|
|
481
|
+
# Yes, this will omit a day or two at the end of the year. But since 365 % 14 == 1, that means only Dec 31st won't
|
|
482
|
+
# get included in the climatology (or Dec 30-31st in a leap year). That is fine, because we're going to interpolate
|
|
483
|
+
# between the time periods anyway in the main code.
|
|
484
|
+
while end_date.year == year:
|
|
485
|
+
yield start_date, mid_date, end_date
|
|
486
|
+
start_date = end_date
|
|
487
|
+
end_date = calc_end_date(end_date)
|
|
488
|
+
mid_date = calc_mid_date(start_date)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def lat_v_theta_clim_name(year, target_pres, by_hour):
|
|
492
|
+
# Include the target pressure levels in the name to avoid accidentally overwriting files if we want to experiment
|
|
493
|
+
# with different pressure levels.
|
|
494
|
+
if isinstance(target_pres, (int, float)):
|
|
495
|
+
target_pres_name_str = '{}hPa'.format(target_pres)
|
|
496
|
+
else:
|
|
497
|
+
target_pres_name_str = '-'.join(str(p) for p in target_pres) + 'hPa'
|
|
498
|
+
|
|
499
|
+
if by_hour:
|
|
500
|
+
nc_file_name = 'GEOS_FPIT_lat_vs_theta_{}_{}_by_hour.nc'.format(year, target_pres_name_str)
|
|
501
|
+
else:
|
|
502
|
+
nc_file_name = 'GEOS_FPIT_lat_vs_theta_{}_{}.nc'.format(year, target_pres_name_str)
|
|
503
|
+
|
|
504
|
+
return nc_file_name
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def make_geos_lat_v_theta_climatology(year, geos_path, save_path, freq=pd.Timedelta(days=14), by_hour=False,
|
|
508
|
+
skip_if_missing=False, bin_var='latitude', target_pres=700):
|
|
509
|
+
"""
|
|
510
|
+
The main driver function to product the latitude vs. mid-tropospheric potential temperature climatology.
|
|
511
|
+
|
|
512
|
+
:param year: which year to use to generate the climatology
|
|
513
|
+
:type year: int
|
|
514
|
+
|
|
515
|
+
:param geos_path: where the GEOS FP or FP-IT Np (i.e. 3D pressure level files) are stored.
|
|
516
|
+
:type geos_path: str
|
|
517
|
+
|
|
518
|
+
:param save_path: where to save the resulting netCDF file
|
|
519
|
+
:type save_path: str
|
|
520
|
+
|
|
521
|
+
:param freq: how long each averaging time period should be
|
|
522
|
+
:type freq: timedelta-lik
|
|
523
|
+
|
|
524
|
+
:param by_hour: set to True to bin different UTC hours' GEOS files separately. This will make the "hours" dimension
|
|
525
|
+
in the netCDF file be > 1. This was an experimental feature and isn't used in the production climatology.
|
|
526
|
+
:type by_hour: bool
|
|
527
|
+
|
|
528
|
+
:param skip_if_missing: set to ``True`` to simply omit time periods missing one or more GEOS files from the
|
|
529
|
+
climatology rather than raising an error. This is intended for testing this code with a subset of GEOS data and
|
|
530
|
+
definitely should not be used in production.
|
|
531
|
+
:type skip_if_missing: bool
|
|
532
|
+
|
|
533
|
+
:param bin_var: which variable to use to define the bins. May be 'latitude' or 'theta'. Change this to 'theta' at
|
|
534
|
+
your own risk, as it will cause northern and southern latitudes to get binned together, which makes no sense.
|
|
535
|
+
Further, the main prior generating code expects the data to have been binned by latitude.
|
|
536
|
+
:type bin_var: str
|
|
537
|
+
|
|
538
|
+
:param target_pres: the pressure or pressures to calculate theta on. If given as a single number, the level of
|
|
539
|
+
``theta`` use for the theta bins will be the level closest to it. If given as a two-element collection (e.g.
|
|
540
|
+
``[500, 700]``), then all layers in theta between those values (inclusive) will be averaged along the vertical
|
|
541
|
+
dimension before calculating the bins. In the second form, order does not matter (the lesser or greater number
|
|
542
|
+
may come first).
|
|
543
|
+
:type target_pres: int, float, or list(int or float)
|
|
544
|
+
|
|
545
|
+
:return: None, saves netCDF file
|
|
546
|
+
"""
|
|
547
|
+
if not by_hour:
|
|
548
|
+
hours = (None,)
|
|
549
|
+
else:
|
|
550
|
+
hours = list(range(0, 24, 3))
|
|
551
|
+
|
|
552
|
+
# The stats will be generated in several loops, one over time periods, and one over hours. As we finish each loop,
|
|
553
|
+
# the lists from that loop will get concatenated into single arrays with a new first dimension.
|
|
554
|
+
all_bin_centers = []
|
|
555
|
+
all_theta_stats = []
|
|
556
|
+
all_lat_stats = []
|
|
557
|
+
dates = []
|
|
558
|
+
|
|
559
|
+
for start, midpoint, end in iter_time_periods(year, freq):
|
|
560
|
+
bin_centers = []
|
|
561
|
+
theta_stats = []
|
|
562
|
+
lat_stats = []
|
|
563
|
+
do_save = True
|
|
564
|
+
for hr in hours:
|
|
565
|
+
# Usually this will just loop once, but it's in here as an experimental option to treat each UTC time
|
|
566
|
+
# separately. This never ended up being used, even in testing.
|
|
567
|
+
lat, plevs, theta = load_geos_data(geos_path, start, end, hours=(hr,), skip_if_missing=skip_if_missing)
|
|
568
|
+
if lat is None:
|
|
569
|
+
do_save = False
|
|
570
|
+
continue
|
|
571
|
+
this_bin_centers, this_theta_stats, this_lat_stats, percentiles = bin_lat_vs_theta(lat, theta, plevs,
|
|
572
|
+
bin_var=bin_var, target_pres=target_pres)
|
|
573
|
+
bin_centers.append(this_bin_centers)
|
|
574
|
+
theta_stats.append(this_theta_stats)
|
|
575
|
+
lat_stats.append(this_lat_stats)
|
|
576
|
+
|
|
577
|
+
if do_save:
|
|
578
|
+
# Concatenate the individual hours in the theta_stats, lat_stats, and bin_centers lists into single arrays
|
|
579
|
+
# that are (nhours)x(nbins)x(nvalues)
|
|
580
|
+
all_bin_centers.append(_cat_stats(bin_centers))
|
|
581
|
+
all_theta_stats.append(_cat_stats(theta_stats))
|
|
582
|
+
all_lat_stats.append(_cat_stats(lat_stats))
|
|
583
|
+
dates.append((start, midpoint, end))
|
|
584
|
+
|
|
585
|
+
# Convert to (ndays)x(nhours)x(nbins)x(nvalues)
|
|
586
|
+
all_bin_centers = _cat_stats(all_bin_centers)
|
|
587
|
+
all_theta_stats = _cat_stats(all_theta_stats)
|
|
588
|
+
all_lat_stats = _cat_stats(all_lat_stats)
|
|
589
|
+
|
|
590
|
+
nc_file_name = lat_v_theta_clim_name(year, target_pres, by_hour)
|
|
591
|
+
nc_file_name = os.path.join(save_path, nc_file_name)
|
|
592
|
+
save_bin_ncdf_file(nc_file_name, dates, hours, bin_var, all_bin_centers, all_theta_stats, all_lat_stats,
|
|
593
|
+
percentiles=percentiles, target_pres=target_pres)
|
|
594
|
+
|