ginput 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. ginput/__init__.py +4 -0
  2. ginput/common_utils/__init__.py +0 -0
  3. ginput/common_utils/ggg_logging.py +62 -0
  4. ginput/common_utils/ioutils.py +304 -0
  5. ginput/common_utils/mod_constants.py +24 -0
  6. ginput/common_utils/mod_utils.py +2663 -0
  7. ginput/common_utils/readers.py +477 -0
  8. ginput/common_utils/run_utils.py +57 -0
  9. ginput/common_utils/sat_utils.py +67 -0
  10. ginput/common_utils/versioning.py +251 -0
  11. ginput/common_utils/writers.py +377 -0
  12. ginput/data/.gitignore +1 -0
  13. ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
  14. ginput/data/ML_monthly_obs_ch4.txt +498 -0
  15. ginput/data/ML_monthly_obs_co.txt +412 -0
  16. ginput/data/ML_monthly_obs_co2.txt +597 -0
  17. ginput/data/ML_monthly_obs_n2o.txt +330 -0
  18. ginput/data/SMO_monthly_obs_ch4.txt +499 -0
  19. ginput/data/SMO_monthly_obs_co.txt +416 -0
  20. ginput/data/SMO_monthly_obs_co2.txt +613 -0
  21. ginput/data/SMO_monthly_obs_n2o.txt +330 -0
  22. ginput/data/ace_fn2o_lut.nc +0 -0
  23. ginput/data/age_spectra/age.midlat.txt +47 -0
  24. ginput/data/age_spectra/age.tropics.txt +46 -0
  25. ginput/data/age_spectra/age.vortex.txt +31 -0
  26. ginput/data/age_spectra/agespec.midlat.txt +47 -0
  27. ginput/data/age_spectra/agespec.tropics.txt +46 -0
  28. ginput/data/age_spectra/agespec.vortex.txt +31 -0
  29. ginput/data/age_spectra/time.txt +3001 -0
  30. ginput/data/ch4_hf_slopes.nc +0 -0
  31. ginput/data/ch4_strat_lut.nc +0 -0
  32. ginput/data/clams_age_clim_scaled.nc +0 -0
  33. ginput/data/co2_strat_lut.nc +0 -0
  34. ginput/data/fo2_data/.gitignore +2 -0
  35. ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
  36. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
  37. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
  38. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
  39. ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
  40. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
  41. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
  42. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
  43. ginput/data/hf_strat_lut.nc +0 -0
  44. ginput/data/isotopologs.dat +199 -0
  45. ginput/data/meso_co_lut.nc +0 -0
  46. ginput/data/n2o_ch4_acefts.nc +0 -0
  47. ginput/data/n2o_strat_lut.nc +0 -0
  48. ginput/data/o2_mean_dmf.dat +52 -0
  49. ginput/data/summer_35N.vmr +182 -0
  50. ginput/data/washenfelder03_table_s3.txt +83 -0
  51. ginput/download/__init__.py +0 -0
  52. ginput/download/download_utils.py +46 -0
  53. ginput/download/get_GEOS5.py +367 -0
  54. ginput/download/get_MERRA2.py +177 -0
  55. ginput/download/get_NOAA_flask_data.py +103 -0
  56. ginput/download/get_fo2_data.py +140 -0
  57. ginput/mod_maker/__init__.py +0 -0
  58. ginput/mod_maker/mod_maker.py +2470 -0
  59. ginput/mod_maker/slantify.py +450 -0
  60. ginput/mod_maker/tccon_sites.py +570 -0
  61. ginput/priors/__init__.py +0 -0
  62. ginput/priors/acos_interface.py +1376 -0
  63. ginput/priors/automation.py +313 -0
  64. ginput/priors/backend_analysis/__init__.py +0 -0
  65. ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
  66. ginput/priors/backend_analysis/ace_fts_val.py +452 -0
  67. ginput/priors/backend_analysis/acos_comparison.py +550 -0
  68. ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
  69. ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
  70. ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
  71. ginput/priors/backend_analysis/backend_driver.py +90 -0
  72. ginput/priors/backend_analysis/backend_utils.py +358 -0
  73. ginput/priors/backend_analysis/clams.py +71 -0
  74. ginput/priors/backend_analysis/create_test_priors.py +853 -0
  75. ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
  76. ginput/priors/backend_analysis/mls_val.py +217 -0
  77. ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
  78. ginput/priors/fo2_prep.py +311 -0
  79. ginput/priors/map_maker.py +193 -0
  80. ginput/priors/mlo_smo_prep.py +1146 -0
  81. ginput/priors/tccon_priors.py +3482 -0
  82. ginput-1.3.0.dist-info/LICENSE.txt +203 -0
  83. ginput-1.3.0.dist-info/METADATA +125 -0
  84. ginput-1.3.0.dist-info/RECORD +86 -0
  85. ginput-1.3.0.dist-info/WHEEL +5 -0
  86. ginput-1.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1418 @@
1
+ from __future__ import print_function, division, absolute_import
2
+
3
+ from collections import OrderedDict
4
+ import datetime as dt
5
+ from glob import glob
6
+ from itertools import repeat
7
+ from matplotlib import pyplot as plt
8
+ from multiprocessing import Pool
9
+ import netCDF4 as ncdf
10
+ import numpy as np
11
+ import pandas as pd
12
+ import os
13
+ import re
14
+ from scipy.optimize import curve_fit
15
+ from statsmodels.robust.robust_linear_model import RLM
16
+ from statsmodels.robust.norms import TukeyBiweight
17
+ import xarray as xr
18
+
19
+ from ...common_utils import mod_utils, ioutils, sat_utils
20
+ from ...common_utils.ggg_logging import logger
21
+ from .. import tccon_priors
22
+ from ...mod_maker import mod_maker as mm
23
+ from .backend_utils import read_ace_var, read_ace_date, read_ace_theta, read_ace_latlon
24
+
25
+ _mydir = os.path.abspath(os.path.dirname(__file__))
26
+ _tccon_top_alt = 70.0
27
+
28
+
29
+ def _bin_centers(bins):
30
+ return bins[:-1] + np.diff(bins)/2
31
+
32
+
33
+ _std_age_bins = np.arange(0.0, 10.0, 0.25)
34
+ _std_age_bin_centers = _bin_centers(_std_age_bins)
35
+ _std_frac_bins = np.arange(0.0, 1.05, 0.05)
36
+ _std_frac_bin_centers = _bin_centers(_std_frac_bins)
37
+ _std_theta_bins = np.concatenate([np.arange(380, 1080, 50), np.arange(1080, 1680, 100), np.arange(1680, 3680, 200)])
38
+ _std_theta_bin_centers = _bin_centers(_std_theta_bins)
39
+
40
+
41
+ def ace_lut_driver(ace_dir, rw_supp_table_file, geos_path, lut_save_dir, overwrite_age_file=True, nprocs=0):
42
+ """Generate all the ACE-derived look-up tables needed for ginput
43
+
44
+ Parameters
45
+ ----------
46
+ ace_dir : str
47
+ Directory to the ACE-FTS data. Expects the ACEs files therein to be named "ACE*{specie}.nc", e.g.
48
+ "ACEFTS_L2_v3p6_N2O.nc".
49
+
50
+ rw_supp_table_file : str
51
+ Path to text file containing Table S3.1 from Washenfelder et al. 2003 (doi:10.1029/2003GL017969). This is
52
+ usually "washenfelder03_table_s3.txt" in ginput/data.
53
+
54
+ geos_path : str
55
+ Path to the GEOS-FPIT met directory containing the "Nv" subdirectory with eta-level GEOS met files.
56
+
57
+ lut_save_dir : str
58
+ Path to save the newly generated LUT files to
59
+
60
+ overwrite_age_file : bool
61
+ Whether or not to recreate and overwrite the ACE-FTS "AGE" file if it exists.
62
+
63
+ nprocs : int
64
+ Number of processors to use when generating the ACE AGE file.
65
+
66
+ Returns
67
+ -------
68
+ None
69
+ Writes 3 files (ace_fn2o_lut.nc, n2o_ch4_acefts.nc, and ch4_hf_slopes.nc) to `lut_save_dir` and an ACE "AGE"
70
+ file to `ace_dir` if either that file does not exist or `overwrite_age_file` is `True`.
71
+
72
+ """
73
+ # Make a dictionary of the available ACE files by species
74
+ ace_files = {re.search(r'[A-Z0-9]+(?=\.nc)', os.path.basename(f)).group(): f for f in glob(os.path.join(ace_dir, 'ACE*.nc'))}
75
+
76
+ if 'AGE' not in ace_files or overwrite_age_file:
77
+ age_file = re.sub(r'P.nc$', 'AGE.nc', ace_files['P'])
78
+ logger.info('Creating ACE-FTS AGE file. Will save to {}. Using {} processors'.format(age_file, nprocs))
79
+ generate_ace_age_file(ace_in_file=ace_files['P'], age_file_out=age_file, geos_path=geos_path,
80
+ use_geos_theta_for_age=False, nprocs=nprocs)
81
+ ace_files['AGE'] = age_file
82
+
83
+ fn2o_save_file = os.path.join(lut_save_dir, 'ace_fn2o_lut.nc')
84
+ logger.info('Creating F(N2O) look up table file. Will save to {}.'.format(fn2o_save_file))
85
+ make_fn2o_lookup_table(ace_age_file=ace_files['AGE'], ace_n2o_file=ace_files['N2O'], lut_save_file=fn2o_save_file)
86
+
87
+ fch4_save_file = os.path.join(lut_save_dir, 'n2o_ch4_acefts.nc')
88
+ logger.info('Creating F(CH4):F(N2O) look up table file. Will save to {}.'.format(fch4_save_file))
89
+ make_fch4_fn2o_lookup_table(ace_age_file=ace_files['AGE'], ace_n2o_file=ace_files['N2O'], ace_ch4_file=ace_files['CH4'],
90
+ lut_save_file=fch4_save_file)
91
+
92
+ hf_slope_save_file = os.path.join(lut_save_dir, 'ch4_hf_slopes.nc')
93
+ logger.info('Creating CH4:HF slope look up table file. Will save to {}'.format(hf_slope_save_file))
94
+ make_hf_ch4_slopes(ace_ch4_file=ace_files['CH4'], ace_hf_file=ace_files['HF'], ace_age_file=ace_files['AGE'],
95
+ washenfelder_supp_table_file=rw_supp_table_file, lut_save_file=hf_slope_save_file,
96
+ clip_values=True)
97
+
98
+
99
+ def make_fn2o_lookup_table(ace_age_file, ace_n2o_file, lut_save_file):
100
+ logger.info('Loading ACE data')
101
+ ace_alt, ace_fn2o, ace_theta, extra_info = calc_fraction_remaining_from_acefts(ace_n2o_file, 'N2O', bc_approach='fit')
102
+
103
+ ace_dates = extra_info['dates']
104
+ ace_doy = np.array([int(mod_utils.clams_day_of_year(date)) for date in ace_dates.flat]).reshape(ace_dates.shape)
105
+
106
+ with ncdf.Dataset(ace_n2o_file, 'r') as nch:
107
+ ace_lat = read_ace_var(nch, 'latitude', None)
108
+
109
+ with ncdf.Dataset(ace_age_file, 'r') as nch:
110
+ ace_age = nch.variables['age'][:].filled(np.nan)
111
+
112
+ logger.info('Binning F(N2O) vs. age and theta')
113
+ # Similar to the F(CH4) vs. F(N2O) approach, we ignore fill values, want only positive F(N2O) values, want to
114
+ # avoid the polar vortex and want to stay within TCCON relevant altitudes.
115
+ #
116
+ # is_vortex expects everything to be the same size, so we need to tile lat and doy
117
+ ace_lat = np.tile(ace_lat[:, np.newaxis], [1, ace_age.shape[1]])
118
+ ace_doy = np.tile(ace_doy[:, np.newaxis], [1, ace_age.shape[1]])
119
+
120
+ xx = ~np.isnan(ace_fn2o) & ~np.isnan(ace_age) & (ace_fn2o >= 0) & \
121
+ ~mod_utils.is_vortex(ace_lat, ace_doy, ace_age) & (ace_alt[np.newaxis, :] < _tccon_top_alt)
122
+
123
+ age_bins = _std_age_bins
124
+ age_bin_centers = _bin_centers(age_bins)
125
+ theta_bins = _std_theta_bins
126
+ theta_bin_centers = _bin_centers(theta_bins)
127
+
128
+ # We're not going to worry about outliers initially
129
+ fn2o_means, fn2o_counts, fn2o_overall, theta_overall = _bin_z_vs_xy(ace_age, ace_theta, ace_fn2o, xx, age_bins, theta_bins)
130
+
131
+ fn2o_means[fn2o_means > 1] = 1
132
+ fn2o_overall[fn2o_overall > 1] = 1
133
+
134
+ _save_fn2o_lut(lut_save_file, fn2o_means=fn2o_means, fn2o_counts=fn2o_counts, fn2o_overall=fn2o_overall,
135
+ theta_overall=theta_overall, age_bin_centers=age_bin_centers, age_bin_edges=age_bins,
136
+ theta_bin_centers=theta_bin_centers, theta_bin_edges=theta_bins,
137
+ ace_n2o_file=ace_n2o_file, ace_age_file=ace_age_file)
138
+
139
+
140
+ def make_fch4_fn2o_lookup_table(ace_age_file, ace_n2o_file, ace_ch4_file, lut_save_file):
141
+ logger.info('Reading ACE data')
142
+ ace_alt, ace_fn2o, ace_theta, extra_info = calc_fraction_remaining_from_acefts(ace_n2o_file, 'N2O', bc_approach='fit')
143
+ ace_dates = extra_info['dates']
144
+ ace_doy = np.array([int(mod_utils.clams_day_of_year(date)) for date in ace_dates.flat]).reshape(ace_dates.shape)
145
+
146
+ _, ace_fch4, _, _ = calc_fraction_remaining_from_acefts(ace_ch4_file, 'CH4', bc_approach='fit')
147
+
148
+ with ncdf.Dataset(ace_age_file, 'r') as nch:
149
+ ace_age = nch.variables['age'][:].filled(np.nan)
150
+
151
+ with ncdf.Dataset(ace_ch4_file, 'r') as nch:
152
+ ace_ch4_raw = nch.variables['CH4'][:].filled(np.nan)
153
+ ace_lat = nch.variables['latitude'][:].filled(np.nan)
154
+
155
+ logger.info('Binning F(CH4) vs. F(N2O)')
156
+ # We're looking for reliable stratospheric relationships. Therefore we limit to data where the concentration is
157
+ # positive and definitely not tropospheric (CH4 < 2e-6 i.e. < 2000 ppb), not in the polar vortex (abs(lat) < 50)
158
+ # and not in the mesosphere or upper stratosphere (alt < 40).
159
+ ace_lat = np.tile(ace_lat[:, np.newaxis], [1, ace_age.shape[1]])
160
+ ace_doy = np.tile(ace_doy[:, np.newaxis], [1, ace_age.shape[1]])
161
+ xx = ~np.isnan(ace_fch4) & ~np.isnan(ace_fn2o) & (ace_fch4 >= 0) & (ace_fn2o >= 0) & (ace_ch4_raw < 2e-6) & \
162
+ ~mod_utils.is_vortex(ace_lat, ace_doy, ace_age) & (ace_alt[np.newaxis, :] < _tccon_top_alt)
163
+
164
+ # Define bins for F(N2O) and theta
165
+ fn2o_bins = _std_frac_bins
166
+ fn2o_bin_centers = _std_frac_bin_centers
167
+
168
+ theta_bins = _std_theta_bins
169
+ theta_bin_centers = _std_theta_bin_centers
170
+
171
+ # Find outliers to avoid overly noisy relationships
172
+ oo = _find_ch4_outliers(ace_fn2o, ace_fch4, fn2o_bins, xx)
173
+
174
+ # Do the actual binning
175
+ fch4_means, fch4_counts, fch4_overall, theta_overall = _bin_z_vs_xy(ace_fn2o, ace_theta, ace_fch4, xx & oo,
176
+ fn2o_bins, theta_bins)
177
+
178
+ logger.info('Saving F(CH4):F(N2O) file')
179
+ _save_fch4_lut(lut_save_file, fch4_means=fch4_means, fch4_counts=fch4_counts, fch4_overall=fch4_overall,
180
+ theta_overall=theta_overall, fn2o_bin_centers=fn2o_bin_centers, fn2o_bin_edges=fn2o_bins,
181
+ theta_bin_centers=theta_bin_centers, theta_bin_edges=theta_bins,
182
+ ace_ch4_file=ace_ch4_file, ace_n2o_file=ace_n2o_file, ace_age_file=ace_age_file)
183
+
184
+
185
+ def make_hf_ch4_slopes(ace_ch4_file, ace_hf_file, ace_age_file, washenfelder_supp_table_file, lut_save_file,
186
+ clip_values=True):
187
+
188
+ logger.info('Loading ACE data')
189
+ with ncdf.Dataset(ace_ch4_file, 'r') as nch_ch4, ncdf.Dataset(ace_hf_file, 'r') as nch_hf, ncdf.Dataset(ace_age_file, 'r') as nch_age:
190
+ ace_ch4_qual = nch_ch4.variables['quality_flag'][:].filled(9)
191
+ ace_hf_qual = nch_hf.variables['quality_flag'][:].filled(9)
192
+
193
+ ace_ch4 = read_ace_var(nch_ch4, 'CH4', ace_ch4_qual)
194
+ ace_ch4_err = read_ace_var(nch_ch4, 'CH4_error', ace_ch4_qual)
195
+ ace_hf = read_ace_var(nch_hf, 'HF', ace_hf_qual)
196
+ ace_hf_err = read_ace_var(nch_hf, 'HF_error', ace_hf_qual)
197
+ ace_ages = read_ace_var(nch_age, 'age', None)
198
+
199
+ ace_lat = read_ace_var(nch_ch4, 'latitude', None)
200
+ ace_lat = np.tile(ace_lat.reshape(-1, 1), [1, ace_ch4.shape[1]])
201
+ ace_alt = read_ace_var(nch_ch4, 'altitude', None)
202
+ ace_alt = np.tile(ace_alt, [ace_ch4.shape[0], 1])
203
+ ace_theta = read_ace_theta(nch_ch4, ace_ch4_qual)
204
+
205
+ ace_year = nch_ch4.variables['year'][:].filled(np.nan)
206
+ ace_dates = read_ace_date(nch_ch4)
207
+ ace_doy = np.array([mod_utils.day_of_year(d) + 1 for d in ace_dates])
208
+ ace_doy = ace_doy.reshape(-1, 1)
209
+
210
+ logger.info('Calculating CH4 vs. HF slopes')
211
+ # We do the same filtering as for the F(N2O):F(CH4) relationship, plus we require that the CH4 and HF error is < 5%.
212
+ # Also have to filter for excessively high HF concentrations, or it will mess up the tropics. Generally HF doesn't
213
+ # exceed 3 ppb and the ultra high values were all > 200 ppb, so 10 should leave all reasonable data and exclude
214
+ # unreasonable data (this occurred in tropics 2005)
215
+ #
216
+ # Quality filtering and fill values were handled upon reading in the data.
217
+ xx = ~np.isnan(ace_ch4) & ~np.isnan(ace_ch4_err) & ~np.isnan(ace_hf) & ~np.isnan(ace_hf_err) & (ace_alt < _tccon_top_alt)
218
+
219
+ if clip_values:
220
+ xx &= (ace_ch4 <= 2e-6) & (ace_hf < 10e-9)
221
+
222
+ # Read in the early slopes from Washenfelder et al. 2003 (doi: 10.1029/2003GL017969), table S3.
223
+ washenfelder_df = pd.read_csv(washenfelder_supp_table_file, header=2, sep=r'\s+')
224
+ washenfelder_slopes = washenfelder_df.set_index('Date').b.astype(float)
225
+ washenfelder_slopes.index = pd.DatetimeIndex(washenfelder_slopes.index)
226
+ washenfelder_slopes.name = 'slope'
227
+
228
+ # For each bin, get the slopes from the ACE-FTS data. Fit the data (including the Washenfelder slopes) to an
229
+ # exponential.
230
+ lat_bin_functions = OrderedDict([('tropics', mod_utils.is_tropics),
231
+ ('midlat', mod_utils.is_midlat),
232
+ ('vortex', mod_utils.is_vortex)])
233
+ lat_bin_slopes = pd.DataFrame(index=_get_ace_date_range(ace_dates), columns=lat_bin_functions.keys())
234
+ lat_bin_counts = lat_bin_slopes.copy()
235
+
236
+ fit_params = np.zeros([len(lat_bin_functions), 4], dtype=float)
237
+
238
+ for ibin, (bin_name, bin_fxn) in enumerate(lat_bin_functions.items()):
239
+ for year_start_date in lat_bin_slopes.index:
240
+ year = year_start_date.year
241
+ logger.debug('Generating {} slopes for {}'.format(bin_name, year))
242
+ lat_bin_slopes.loc[year_start_date, bin_name], lat_bin_counts.loc[year_start_date, bin_name] \
243
+ = _bin_and_fit_hf_vs_ch4(ace_lat=ace_lat, ace_year=ace_year, ace_dates=ace_dates, ace_doy=ace_doy,
244
+ ace_ages=ace_ages, ace_theta=ace_theta, ace_ch4=ace_ch4, ace_hf=ace_hf, xx=xx,
245
+ year=year, bin_fxn=bin_fxn)
246
+
247
+ full_series = pd.concat([washenfelder_slopes, lat_bin_slopes.loc[:, bin_name]])
248
+ full_series = full_series.sort_index()
249
+ full_series_year = full_series.index.year
250
+ full_series = full_series.to_numpy().astype(float)
251
+
252
+ # This was found to be a good initial guess for all latitude bins by trial and error. 3000.0 gets about the
253
+ # right magnitude at year 0, 0.18 get about the right decay, using the last slope as the offset generally works
254
+ # better than using the maximum (sometimes curve_fit has trouble moving this offset enough) and naturally we
255
+ # want t0 to be the first year. Trying to allow curve_fit to choose its own initial conditions ends badly.
256
+ p0 = [-3000.0, -0.18, full_series[-1], full_series_year[0]]
257
+ fit = curve_fit(mod_utils.hf_ch4_slope_fit, full_series_year, full_series, p0=p0)
258
+ fit_params[ibin, :] = fit[0]
259
+
260
+ # for the netCDF file, we'll include both the fits and the actual slopes for the ACE-FTS era. The default is to
261
+ # use the fits for all years.
262
+ first_year = washenfelder_slopes.index.min().year
263
+ last_year = lat_bin_slopes.index.max().year
264
+ full_dtindex = pd.date_range(start=dt.datetime(first_year, 1, 1), end=dt.datetime(last_year, 1, 1), freq='YS')
265
+
266
+ logger.info('Saving CH4 vs. HF slopes')
267
+ _save_hf_ch4_lut(lut_save_file, ace_ch4_file=ace_ch4_file, ace_hf_file=ace_hf_file, ace_age_file=ace_age_file,
268
+ lat_bin_edges=lat_bin_functions, full_date_index=full_dtindex,
269
+ ace_slopes=lat_bin_slopes, ace_counts=lat_bin_counts, slope_fit_params=fit_params)
270
+
271
+ return lat_bin_slopes, lat_bin_counts
272
+
273
+
274
+ def _find_ch4_outliers(ace_fn2o, ace_fch4, fn2o_bins, good_data):
275
+
276
+ # Identify outliers in each F(N2O) bin. We're not binning by theta yet; we just want to find outliers in the dataset
277
+ # as a whole right now. I chose m=5 since 5 sigma is the standard threshold in physics, even through we're using
278
+ # mean absolute deviation and not standard deviation, since the former is more robust.
279
+ #
280
+ # oo will be true for non-outlier values
281
+ oo = np.zeros_like(ace_fch4, dtype=np.bool_)
282
+ bin_inds = np.digitize(ace_fn2o, fn2o_bins) - 1
283
+ for b_ind in np.unique(bin_inds):
284
+ bb = good_data & (bin_inds == b_ind)
285
+ ch4_sub = ace_fch4[bb]
286
+ oo[bb] = ~mod_utils.isoutlier(ch4_sub, m=5)
287
+
288
+ return oo
289
+
290
+
291
+ def _bin_z_vs_xy(x, y, z, good_data, x_bins, y_bins):
292
+ n_x_bins = x_bins.size - 1
293
+ n_y_bins = y_bins.size - 1
294
+ x_means = np.full([n_x_bins, n_y_bins], np.nan)
295
+ x_counts = np.zeros_like(x_means, dtype=int)
296
+
297
+ for i, (tlow, thigh) in enumerate(zip(y_bins[:-1], y_bins[1:])):
298
+ aa = good_data & (y >= tlow) & (y < thigh)
299
+ x_sub = x[aa]
300
+ z_sub = z[aa]
301
+ bin_inds = np.digitize(x_sub, x_bins) - 1
302
+
303
+ for j in range(n_x_bins):
304
+ jj = bin_inds == j
305
+ x_means[j, i] = np.nanmean(z_sub[jj])
306
+ x_counts[j, i] = np.sum(jj)
307
+
308
+ z_overall = np.full([n_x_bins], np.nan)
309
+ y_overall = np.full([n_x_bins], np.nan)
310
+ bin_inds = np.digitize(x, x_bins) - 1
311
+
312
+ for j in range(n_x_bins):
313
+ jj = (bin_inds == j) & good_data
314
+ z_overall[j] = np.nanmean(z[jj])
315
+ y_overall[j] = np.nanmean(y[jj])
316
+
317
+ return x_means, x_counts, z_overall, y_overall
318
+
319
+
320
+ def _bin_and_fit_hf_vs_ch4(ace_lat, ace_year, ace_dates, ace_doy, ace_ages, ace_theta, ace_ch4, ace_hf, xx, year,
321
+ bin_fxn):
322
+ yy = bin_fxn(ace_lat, ace_doy, ace_ages) & (np.isclose(ace_year, year)).reshape(-1, 1)
323
+
324
+ profile_ch4_sbc = get_bc_from_ace(ace_dates=ace_dates, gas_conc=ace_ch4, theta=ace_theta, latitude=ace_lat[:, 0],
325
+ bc_approach='fit')
326
+
327
+ x = ace_hf[xx & yy] * 1e9
328
+ y = (ace_ch4 - profile_ch4_sbc.reshape(-1, 1))
329
+ y = y[xx & yy] * 1e9
330
+
331
+ # We use the robust fitting method with Tukey's biweighting as in Saad et al. 2014. That paper used `robustfit` from
332
+ # Matlab, RLM from statsmodels is the Python equivalent. We do not include an intercept because we are using the
333
+ # CH4 stratospheric boundary condition as the intercept, and we've already subtracted that.
334
+ slope = RLM(y, x, M=TukeyBiweight()).fit().params.item()
335
+
336
+ return slope, np.sum(xx & yy)
337
+
338
+
339
+ def _get_year_avg_ch4_sbc(year, ch4):
340
+ tt = (ch4.conc_seasonal.index >= dt.datetime(year, 1, 1)) & (ch4.conc_seasonal.index < dt.datetime(year + 1, 1, 1))
341
+ return ch4.conc_seasonal.dmf_mean[tt].mean()
342
+
343
+
344
+ def _save_fn2o_lut(nc_filename, fn2o_means, fn2o_counts, fn2o_overall, theta_overall,
345
+ age_bin_centers, age_bin_edges, theta_bin_centers, theta_bin_edges,
346
+ ace_n2o_file, ace_age_file):
347
+
348
+ with ncdf.Dataset(nc_filename, 'w') as nch:
349
+ age_dim = ioutils.make_ncdim_helper(nch, 'age', age_bin_centers,
350
+ description='Age of stratospheric air from the CLaMS model',
351
+ units='years')
352
+ ioutils.make_ncdim_helper(nch, 'age_bins', age_bin_edges,
353
+ description='Edges of age bins used when binning the F(N2O) data',
354
+ units='years')
355
+ theta_dim = ioutils.make_ncdim_helper(nch, 'theta', theta_bin_centers,
356
+ description='Potential temperature',
357
+ units='K')
358
+ ioutils.make_ncdim_helper(nch, 'theta_bins', theta_bin_edges,
359
+ description='Edges of potential temperature bins used when binning the F(N2O) data',
360
+ units='K')
361
+
362
+ ioutils.make_ncvar_helper(nch, 'fn2o', fn2o_means, (age_dim, theta_dim),
363
+ description='Mean value of F(N2O) (fraction of N2O remaining relative to the '
364
+ 'stratospheric boundary condition) in the age/theta bin',
365
+ units='unitless')
366
+ ioutils.make_ncvar_helper(nch, 'fn2o_counts', fn2o_counts, (age_dim, theta_dim),
367
+ description='Number of F(N2O) observations in each age/theta bin',
368
+ units='number')
369
+ ioutils.make_ncvar_helper(nch, 'fn2o_overall', fn2o_overall, (age_dim,),
370
+ description='Mean value of F(N2O) for a given age bin, not separated by theta',
371
+ units='unitless')
372
+ ioutils.make_ncvar_helper(nch, 'theta_overall', theta_overall, (age_dim,),
373
+ description='Mean value of potential temperature in a given age bin',
374
+ units='K')
375
+ ioutils.add_dependent_file_hash(nch, 'ace_n2o_file_sha1', ace_n2o_file)
376
+ ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
377
+ ioutils.add_creation_info(nch, 'make_fn2o_lookup_table')
378
+
379
+
380
+ def _save_fch4_lut(nc_filename, fch4_means, fch4_counts, fch4_overall, theta_overall,
381
+ fn2o_bin_centers, fn2o_bin_edges, theta_bin_centers, theta_bin_edges,
382
+ ace_ch4_file, ace_n2o_file, ace_age_file):
383
+ with ncdf.Dataset(nc_filename, 'w') as nch:
384
+ fn2o_dim = ioutils.make_ncdim_helper(nch, 'fn2o', fn2o_bin_centers,
385
+ description='Fraction of N2O remaining relative to the stratospheric boundary condition.',
386
+ units='unitless')
387
+ ioutils.make_ncdim_helper(nch, 'fn2o_bins', fn2o_bin_edges,
388
+ description='Edges of F(N2O) bins used when binning the F(CH4) data',
389
+ units='unitless')
390
+ theta_dim = ioutils.make_ncdim_helper(nch, 'theta', theta_bin_centers,
391
+ description='Potential temperature',
392
+ units='K')
393
+ ioutils.make_ncdim_helper(nch, 'theta_bins', theta_bin_edges,
394
+ description='Edges of potential temperature bins used when binning the F(CH4) data',
395
+ units='K')
396
+
397
+ ioutils.make_ncvar_helper(nch, 'fch4', fch4_means, (fn2o_dim, theta_dim),
398
+ description='Mean value of F(CH4) (fraction of CH4 remaining relative to the '
399
+ 'stratospheric boundary condition) in the F(N2O)/theta bin',
400
+ units='unitless')
401
+ ioutils.make_ncvar_helper(nch, 'fch4_counts', fch4_counts, (fn2o_dim, theta_dim),
402
+ description='Number of CH4 observations in each F(N2O)/theta bin',
403
+ units='number')
404
+ ioutils.make_ncvar_helper(nch, 'fch4_overall', fch4_overall, (fn2o_dim,),
405
+ description='Mean value of F(CH4) for a given F(N2O) bin, not separated by theta',
406
+ units='unitless')
407
+ ioutils.make_ncvar_helper(nch, 'theta_overall', theta_overall, (fn2o_dim,),
408
+ description='Mean value of potential temperature in a given F(N2O) bin',
409
+ units='K')
410
+ ioutils.add_creation_info(nch, 'make_fch4_fn2o_lookup_table')
411
+ ioutils.add_dependent_file_hash(nch, 'ace_ch4_file_sha1', ace_ch4_file)
412
+ ioutils.add_dependent_file_hash(nch, 'ace_n2o_file_sha1', ace_n2o_file)
413
+ ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
414
+
415
+
416
+ def _save_hf_ch4_lut(nc_filename, ace_ch4_file, ace_hf_file, ace_age_file, lat_bin_edges,
417
+ full_date_index, ace_slopes, ace_counts, slope_fit_params):
418
+ with ncdf.Dataset(nc_filename, 'w') as nch:
419
+ ioutils.add_creation_info(nch, creation_note='make_hf_ch4_slopes')
420
+ ioutils.add_dependent_file_hash(nch, 'ace_ch4_file_sha1', ace_ch4_file)
421
+ ioutils.add_dependent_file_hash(nch, 'ace_hf_file_sha1', ace_hf_file)
422
+ ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
423
+
424
+ nch.ace_ch4_file = ace_ch4_file
425
+ nch.ace_hf_file = ace_hf_file
426
+
427
+ ch4_hf_slopes = pd.DataFrame(index=full_date_index, columns=ace_slopes.columns)
428
+ ch4_hf_count = pd.DataFrame(index=full_date_index, columns=ace_counts.columns).fillna(0)
429
+ ch4_hf_source = pd.DataFrame(index=full_date_index, columns=ace_slopes.columns).fillna(1)
430
+ # transpose so that the bins dim is second like the above dataframes
431
+ bin_names_char_array = ncdf.stringtochar(np.array(list(lat_bin_edges.keys()))).T
432
+ slope_fit_params = slope_fit_params.T
433
+
434
+ # For each latitude bin, make a pandas series for the full date index from the fit first, then replace available
435
+ # years with the actual fit
436
+ for ibin, bin_name in enumerate(lat_bin_edges.keys()):
437
+ ch4_hf_slopes.loc[:, bin_name] = mod_utils.hf_ch4_slope_fit(ch4_hf_slopes.index.year, *slope_fit_params[:, ibin])
438
+ ch4_hf_slopes.loc[ace_slopes.index, bin_name] = ace_slopes.loc[:, bin_name]
439
+ ch4_hf_count.loc[ace_counts.index, bin_name] = ace_counts.loc[:, bin_name]
440
+ ch4_hf_source.loc[ace_slopes.index, bin_name] = 0
441
+
442
+ # Setup dimensions.
443
+ nbins = len(lat_bin_edges)
444
+ nchars = bin_names_char_array.shape[0]
445
+ nparams = slope_fit_params.shape[0]
446
+ year_dim = ioutils.make_ncdim_helper(nch, 'year', full_date_index.year.to_numpy())
447
+ bins_dim = nch.createDimension('latitude_bins', size=nbins)
448
+ char_dim = nch.createDimension('bin_name_length', size=nchars)
449
+ fit_params_dim = nch.createDimension('exp_fit_params_length', size=nparams)
450
+
451
+ # Save the variables
452
+ ioutils.make_ncvar_helper(nch, 'ch4_hf_slopes', ch4_hf_slopes.to_numpy().astype(float), [year_dim, bins_dim],
453
+ units='ppb CH4/ppb HF',
454
+ description='Slope of stratospheric CH4 concentrations vs HF concentrations from '
455
+ 'ACE-FTS data for specific years and latitude bins')
456
+ ioutils.make_ncvar_helper(nch, 'ace_counts', ch4_hf_count.to_numpy().astype(float), [year_dim, bins_dim],
457
+ units='#',
458
+ description='Number of ACE-FTS observations used to calculate this slope')
459
+ ioutils.make_ncvar_helper(nch, 'ch4_hf_slopes_source', ch4_hf_source.to_numpy().astype(int), [year_dim, bins_dim],
460
+ units='N/A',
461
+ description='Flag indicating how the slope was computed. 0 means taken directly from '
462
+ 'fits of ACE-FTS CH4 vs. HF; 1 means sampled from the exponential fit '
463
+ 'of ACE-FTS + Washenfelder 03 slopes')
464
+ ioutils.make_ncvar_helper(nch, 'slope_fit_params', slope_fit_params, [fit_params_dim, bins_dim],
465
+ description='Fitting parameters a, b, c, and t0 for a function of form '
466
+ '"a * exp(b*(t - t0)) + c" used to fit the temporal trend of CH4 vs. HF '
467
+ 'slopes, where t is the year (e.g. 2004)')
468
+ ioutils.make_ncvar_helper(nch, 'bin_names', bin_names_char_array, [char_dim, bins_dim],
469
+ description='The names used for the latitude bins')
470
+
471
+
472
+ def plot_hf_ch4_slopes(lut_file, years=np.arange(1970, 2019), ax=None):
473
+ if ax is None:
474
+ fig, ax = plt.subplots()
475
+ else:
476
+ fig = ax.figure
477
+ colors = ['r', 'g', 'b']
478
+ linestyles = ['-', '--', ':']
479
+ with xr.open_dataset(lut_file) as ds:
480
+ lat_bins = ds['latitude_bins'][:]
481
+ params = ds['slope_fit_params'][:]
482
+ bin_names = ds['bin_names'][:]
483
+ for i, col, style in zip(lat_bins, colors, linestyles):
484
+ label = bin_names.sel(latitude_bins=i).data
485
+ try:
486
+ label = ''.join(label)
487
+ except TypeError:
488
+ label = b''.join(label).decode('utf8')
489
+
490
+ slopes = mod_utils.hf_ch4_slope_fit(years, *params.sel(latitude_bins=i).data)
491
+ ax.plot(years, slopes, color=col, linestyle=style, label=label)
492
+
493
+ ax.grid()
494
+ ax.set_ylabel('CH4:HF slope from fit')
495
+ ax.legend()
496
+
497
+ return fig, ax
498
+
499
+
500
+ def get_bc_from_ace(ace_dates, gas_conc, theta, latitude, bc_approach='fit'):
501
+ """
502
+ Calcaulate the stratospheric boundary condition for each ACE profile from ACE data itself
503
+
504
+ :param ace_dates: a 1D array of datetimes with one entry per ACE profile
505
+ :type ace_dates: array-like
506
+
507
+ :param gas_conc: the array of ACE concentrations for the gas of interest. May be any units, the boundary condition
508
+ values will be returned in the same units.
509
+ :type gas_conc: array-like
510
+
511
+ :param theta: the array of ACE potential temperatures.
512
+ :type theta: array-like
513
+
514
+ :param latitude: the array of ACE profile latitudes (1D)
515
+ :type latitude: array-like
516
+
517
+ :param bc_approach: how to calculate the boundary condition. "fit" (default) fits all ACE data in the tropics
518
+ between 360 and 390 K to generate a quadratic fit over time. "per-profile" calculates the boundary condition for
519
+ each profile using its own data between 360 and 390 K.
520
+ :type bc_approach: str
521
+
522
+ :return: the array of boundary condition concentrations, one per profile.
523
+ """
524
+ bottom_pt = 360
525
+ top_pt = 390
526
+
527
+ def _get_bc_per_profile(dates, concentration, pt, lat):
528
+ bc_conc = np.full([concentration.shape[0]], np.nan)
529
+ for i, (conc_row, pt_row) in enumerate(zip(concentration, pt)):
530
+ zz = (pt_row > bottom_pt) & (pt_row < top_pt)
531
+
532
+ if zz.sum() > 0:
533
+ bc_conc[i] = np.nanmean(conc_row[zz])
534
+
535
+ return bc_conc
536
+
537
+ def _get_bc_by_fit(dates, concentration, pt, lat):
538
+ datenums = np.full_like(concentration, np.nan)
539
+ datenums[:] = np.array([np.datetime64(d) for d in dates]).reshape(-1, 1)
540
+ in_pt = (pt > 360) & (pt < 390)
541
+ in_tropics = mod_utils.is_tropics(lat, None, None)
542
+ in_both = in_pt & in_tropics.reshape(-1, 1)
543
+ bc_concs = concentration[in_both]
544
+ bc_datenums = datenums[in_both]
545
+ not_outliers = ~mod_utils.isoutlier(bc_concs, m=5)
546
+ not_nans = ~np.isnan(bc_datenums) & ~np.isnan(bc_concs)
547
+ xx = not_outliers & not_nans
548
+ fit = np.polynomial.polynomial.Polynomial.fit(bc_datenums[xx], bc_concs[xx], deg=2)
549
+ return fit(datenums[:, 0])
550
+
551
+ if bc_approach == 'per-profile':
552
+ bc_fxn = _get_bc_per_profile
553
+ elif bc_approach == 'fit':
554
+ bc_fxn = _get_bc_by_fit
555
+ else:
556
+ raise ValueError('bc_approach must be "per-profile" or "fit"')
557
+
558
+ return bc_fxn(ace_dates, gas_conc, theta, latitude)
559
+
560
+
561
+ def calc_fraction_remaining_from_acefts(nc_file, gas_name, bc_approach='per-profile'):
562
+ """
563
+ Calculate the fraction remaining of a gas in the stratosphere from an ACE-FTS netCDF file
564
+
565
+ :param nc_file: the path to the ACE-FTS netCDF file
566
+ :type nc_file: str
567
+
568
+ :param gas_name: the variable name in the netCDF file that holds the gas concentrations
569
+ :type gas_name: str
570
+
571
+ :param gas_record: a subclass instance of :class:`TropicsTraceGasRecord` that provides the stratospheric boundary
572
+ condition for the given gas.
573
+ :type gas_record: :class:`tccon_priors.TropicsTraceGasRecord`
574
+
575
+ :param tropopause_approach: how to find the tropopause. Options are:
576
+
577
+ * 'wmo' - uses the WMO definition, looking for lapse rate < 2 K/km
578
+ * 'theta' - finds the altitude at which potential temperature is 380 K
579
+
580
+ :type tropopause_approach: str
581
+
582
+ :return: the vector of altitudes that the ACE-FTS profiles are defined on and the array of fraction of gas
583
+ remaining. The latter will be set to NaN in the troposphere.
584
+ :rtype: :class:`numpy.ndarray` x2
585
+ """
586
+
587
+ with ncdf.Dataset(nc_file, 'r') as nch:
588
+ alt = read_ace_var(nch, 'altitude', qflags=None)
589
+ latitude = read_ace_var(nch, 'latitude', qflags=None)
590
+ ace_dates = read_ace_date(nch)
591
+ qflags = read_ace_var(nch, 'quality_flag', qflags=None)
592
+
593
+ gas_conc = read_ace_var(nch, gas_name, qflags=qflags)
594
+ # ACE data appears to use -999 as fill value in the concentrations
595
+ gas_conc[gas_conc < -900.0] = np.nan
596
+
597
+ theta = read_ace_theta(nch, qflags)
598
+
599
+ bc_concentrations = get_bc_from_ace(ace_dates, gas_conc, theta, latitude, bc_approach=bc_approach)
600
+ fgas = gas_conc / bc_concentrations.reshape(-1, 1)
601
+ fgas[theta < 380] = np.nan
602
+
603
+ return alt, fgas, theta, {'dates': ace_dates, 'bc_concs': bc_concentrations, 'gas_concs': gas_conc}
604
+
605
+
606
+ def _get_ace_date_range(ace_dates, freq='YS'):
607
+ min_date = np.min(ace_dates)
608
+ if min_date < dt.datetime(min_date.year, 4, 1):
609
+ start_year = min_date.year
610
+ else:
611
+ start_year = min_date.year + 1
612
+
613
+ max_date = np.max(ace_dates)
614
+ if max_date >= dt.datetime(max_date.year, 10, 1):
615
+ end_year = max_date.year
616
+ else:
617
+ end_year = max_date.year - 1
618
+
619
+ return pd.date_range(start=dt.datetime(start_year, 1, 1), end=dt.datetime(end_year, 1, 1), freq=freq)
620
+
621
+
622
+ def add_clams_age_to_file(ace_nc_file, save_nc_file=None):
623
+ """
624
+ Calculate CLaMS age for profiles in an ACE-FTS files. Save either to the given ACE file or a new file.
625
+
626
+ Note: this function does not calculate equivalent latitude for the ACE profiles before computing the age, it simply
627
+ uses the geographic latitude of the ACE profiles.
628
+
629
+ :param ace_nc_file: the ACE-FTS netCDF file to compute ages for.
630
+ :type ace_nc_file: str
631
+
632
+ :param save_nc_file: optional, if given a path to a file, the ages will be saved to this file (which is overwritten)
633
+ along with the altitude dimension. If not given (or ``None``), the ages are added to the file specified by
634
+ ``ace_nc_file``.
635
+ :type save_nc_file: str
636
+
637
+ :return: None
638
+ """
639
+ with ncdf.Dataset(ace_nc_file, 'r') as nch:
640
+ ace_lat = read_ace_var(nch, 'latitude', None)
641
+ ace_alt = read_ace_var(nch, 'altitude', None)
642
+ ace_dates = read_ace_date(nch)
643
+ ace_doy = np.array([mod_utils.day_of_year(d) + 1 for d in ace_dates])
644
+ ace_theta = read_ace_theta(nch, None)
645
+
646
+ ace_ages = np.full_like(ace_theta, np.nan)
647
+ pbar = mod_utils.ProgressBar(ace_theta.shape[0], prefix='Calculating ages for profile', style='counter')
648
+ for iprof in range(ace_theta.shape[0]):
649
+ pbar.print_bar(iprof)
650
+ prof_theta = ace_theta[iprof]
651
+ prof_lat = np.full_like(prof_theta, ace_lat[iprof])
652
+ prof_doy = ace_doy[iprof]
653
+ ace_ages[iprof, :] = tccon_priors.get_clams_age(prof_theta, prof_lat, prof_doy)
654
+
655
+ pbar.finish()
656
+
657
+ if save_nc_file is None:
658
+ logger.info('Added CLaMS age to input file: {}'.format(ace_nc_file))
659
+ with ncdf.Dataset(ace_nc_file, 'a') as nch:
660
+ ioutils.make_ncvar_helper(nch, 'age', ace_ages, ['index', 'altitude'], units='years',
661
+ description='Age from the CLaMS model along the ACE profiles')
662
+ else:
663
+ logger.info('Saving CLaMS age to new file: {}'.format(save_nc_file))
664
+ with ncdf.Dataset(save_nc_file, 'w') as nch:
665
+ index_dim = nch.createDimension('index', ace_ages.shape[0])
666
+ alt_dim = ioutils.make_ncdim_helper(nch, 'altitude', ace_alt, long_name='altitude',
667
+ description='Fixed altitude grid', units='km')
668
+ ioutils.make_ncvar_helper(nch, 'age', ace_ages, [index_dim, alt_dim], units='years', long_name='clams_age',
669
+ description='Age from the CLaMS model along the ACE profiles')
670
+
671
+
672
+ def generate_ace_age_file(ace_in_file, age_file_out, geos_path, use_geos_theta_for_age=False, geos_product='fpit', allow_missing_geos=False, nprocs=0):
673
+ """Create a file that mimics the structure of an ACE-FTS v3 file but stores age of air and equivalent latitude
674
+
675
+ Parameters
676
+ ----------
677
+ ace_in_file: str
678
+ Path to any of the original ACE files. It doesn't need to be any one in particular; all should have the lat,
679
+ lon, pressure, temperature, quality flag, and date information needed. However, if this is for a particular
680
+ gas, you may as well pass that gas's file, in case the quality flags differ.
681
+
682
+ age_file_out: str
683
+ Path to write the resulting file to (the file itself, not a directory).
684
+
685
+ geos_path: str
686
+ Path to the directory containing the GEOS met data. This directory must contain an "Nv" subdirectory with the
687
+ 3D met files.
688
+
689
+ use_geos_theta_for_age: bool
690
+ Set to ``True`` to use potential temperature calculated from the GEOS files, rather than ACE, when determining
691
+ the age of air.
692
+
693
+ geos_product: str
694
+ A string, usually "fpit", "fp", or "it" that describes which GEOS product to read. This is necessary to determine
695
+ the file names.
696
+
697
+ allow_missing_geos: bool
698
+ Set to ``True`` to skip profiles for which the necessary GEOS files are not available. Otherwise, any missing GEOS
699
+ file is an error.
700
+
701
+ nprocs: int
702
+ How many processors to use. <= 1 will run in serial, >= 2 will use multiprocessing to run in parallel.
703
+ """
704
+ # This will need to group ACE profiles by which GEOS files they fall between, read the PV from those files,
705
+ # interpolate to the ACE profile lat/lon/time, generate the EL interpolators, calculate the EL profiles, then
706
+ # finally lookup age from CLAMS.
707
+ logger.info('Generating ACE age file with NATIVE-level GEOS files')
708
+ ace_data = dict()
709
+ with ncdf.Dataset(ace_in_file) as nh:
710
+ ace_data['dates'] = read_ace_date(nh)
711
+ ace_qflags = read_ace_var(nh, 'quality_flag', None)
712
+ # This wraps lons outside [-180,180] and lats outside [-90,90] into
713
+ # those ranges
714
+ ace_data['lon'], ace_data['lat'] = read_ace_latlon(nh, clip=True)
715
+ ace_data['theta'] = read_ace_theta(nh, qflags=ace_qflags)
716
+
717
+ date_groups = _bin_ace_to_geos_times(ace_data['dates'])
718
+
719
+ # Now we need to loop over the date groups and calculate the eq. lat. and age for each group (since each group
720
+ # is the ace profiles in a 3 hour window bracketed by 2 GEOS files). There's usually only a few profiles per group,
721
+ # so it makes more sense to parallelize over groups than profiles within groups.
722
+ #
723
+ # Even though it may be memory intensive, we're going to build the inputs for each call to _calc_el_age_for_ace
724
+ # first then iterate so that we can use the same inputs for serial or parallel mode. We'll build as iterators to
725
+ # limit the memory use as much as possible.
726
+ group_inputs = dict()
727
+ group_inputs['ace_dates'] = (ace_data['dates'][xx] for xx in date_groups.values())
728
+ group_inputs['ace_lons'] = (ace_data['lon'][xx] for xx in date_groups.values())
729
+ group_inputs['ace_lats'] = (ace_data['lat'][xx] for xx in date_groups.values())
730
+ group_inputs['ace_theta'] = (ace_data['theta'][xx, :] for xx in date_groups.values())
731
+ group_inputs['geos_dates'] = ((d, d+dt.timedelta(hours=3)) for d in date_groups.keys())
732
+ group_inputs['geos_path'] = repeat(geos_path)
733
+ group_inputs['use_geos_theta'] = repeat(use_geos_theta_for_age)
734
+ group_inputs['geos_product'] = repeat(geos_product)
735
+ group_inputs['allow_missing'] = repeat(allow_missing_geos)
736
+
737
+ input_order = ('ace_dates', 'ace_lons', 'ace_lats', 'ace_theta', 'geos_dates', 'geos_path', 'use_geos_theta', 'geos_product', 'allow_missing')
738
+ inputs_interator = zip(*[group_inputs[k] for k in input_order])
739
+
740
+ if nprocs <= 1:
741
+ results = []
742
+ for inputs in inputs_interator:
743
+ results.append(_calc_el_age_for_ace(*inputs))
744
+ else:
745
+ with Pool(processes=nprocs) as pool:
746
+ results = pool.starmap(_calc_el_age_for_ace, inputs_interator)
747
+
748
+ # _calc_el_age_for_age returns a dictionary of nprofs-by-nlevels arrays, where nprofs is the number of profiles in
749
+ # the group it was given. results is then a list of these dictionaries, so we need to go through those dictionaries
750
+ # and put their subset of profiles in the right rows in the bigger array
751
+ output_keys = ('EL', 'age')
752
+ ace_outputs = {k: np.full_like(ace_data['theta'], np.nan) for k in output_keys}
753
+ for i, inds in enumerate(date_groups.values()):
754
+ if results[i] is None and allow_missing_geos:
755
+ continue
756
+ elif results[i] is None and not allow_missing_geos:
757
+ raise TypeError('Got a None value when missing GEOS files were forbidden')
758
+
759
+ for k in output_keys:
760
+ ace_outputs[k][inds] = results[i][k]
761
+
762
+ save_ace_age_file(age_file_out, ('longitude', 'latitude', 'orbit', 'year', 'month', 'day', 'hour', 'quality_flag', 'temperature', 'pressure'),
763
+ ace_in_file, ace_outputs['EL'], ace_outputs['age'])
764
+ logger.info('Done generating ACE age file with NATIVE-level GEOS files')
765
+
766
+
767
+ def _calc_el_age_for_ace(ace_dates, ace_lon, ace_lat, ace_theta, geos_dates, geos_path, use_geos_theta_for_age=False, geos_product='fpit', allow_missing=False):
768
+ # Let's get to the point where we load the GEOS data
769
+ date_str = ', '.join(d.strftime('%Y-%m-%d %H:%M') for d in ace_dates)
770
+ logger.info('Calculating eqlat and age for profiles on {}'.format(date_str))
771
+ # Read in the GEOS data, calculating quantities as necessary #
772
+ geos_vars = {'EPV': 1e6, 'T': 1.0, 'DELP': 1.0}
773
+
774
+ try:
775
+ geos_files = [os.path.join(geos_path, 'Nv', mod_utils._format_geosfp_name(geos_product, 'met', 'eta', d)) for d in geos_dates]
776
+ missing_files = ', '.join(f for f in geos_files if not os.path.exists(f))
777
+ if missing_files:
778
+ if allow_missing:
779
+ date_range = f'{min(ace_dates)} to {max(ace_dates)}'
780
+ logger.warn(f'Missing at least one GEOS file for {date_range}: {missing_files}')
781
+ return None
782
+ else:
783
+ raise IOError(f'Missing GEOS files: {missing_files}')
784
+ geos_data_on_std_times = []
785
+ geos_pres = []
786
+ for i, f in enumerate(geos_files):
787
+ this_data = _interp_geos_vars_to_ace_lat_lon(f, geos_vars, ace_lon, ace_lat)
788
+ geos_pres.append(np.fliplr(mod_utils.convert_geos_eta_coord(this_data.pop('DELP'))))
789
+
790
+ # Handle flipping vertically after computing pressure in case the convert function expects the
791
+ # delp vector space-to-surface. Flipping lr instead of ud because the flip has to go along the
792
+ # second dimension
793
+ for k, v in this_data.items():
794
+ this_data[k] = np.fliplr(v)
795
+
796
+ geos_data_on_std_times.append(this_data)
797
+
798
+ # Generate the eq. lat. intepolators and find the eq. lat. profiles on the GEOS levels #
799
+ eqlat_interpolators = mm.equivalent_latitude_functions_from_native_geos_files(geos_files, geos_dates, muted=True)
800
+ for i, geos_data in enumerate(geos_data_on_std_times):
801
+ gdate = geos_dates[i]
802
+ geos_data['PT'] = mod_utils.calculate_potential_temperature(geos_pres[i], geos_data['T'])
803
+ # Calculate the equivalent latitude profiles.
804
+ eqlat = np.full_like(geos_data['T'], np.nan)
805
+ for j in range(ace_lon.size):
806
+ eqlat[j, :] = mod_utils.get_eqlat_profile(eqlat_interpolators[gdate], geos_data['EPV'][j, :], geos_data['PT'][j, :])
807
+ geos_data['EL'] = eqlat
808
+
809
+ # Put the profiles onto the ACE times and levels #
810
+ geos_data_at_ace_times = _interp_geos_vars_to_ace_times(ace_dates, geos_dates, geos_data_on_std_times)
811
+ geos_data_on_ace_levels = dict()
812
+ for varname, geos_data in geos_data_at_ace_times.items():
813
+ ace_profiles = np.full_like(ace_theta, np.nan)
814
+ for i, _ in enumerate(geos_data):
815
+ # Limit to GEOS levels where potential temperature is monotonically increasing.
816
+ decr_theta = np.diff(geos_data_at_ace_times['PT'][i,:]) <= 0
817
+ if np.any(decr_theta):
818
+ pt_incr = np.flatnonzero(decr_theta)[-1] + 1
819
+ logger.debug('Limiting GEOS data to level {} and up to ensure monotonically increasing potential temperature'.format(pt_incr+1))
820
+ else:
821
+ pt_incr = 0
822
+ logger.debug('GEOS potential temperature monotonically increasing for all levels')
823
+ # Check that the GEOS data still covers the range of ACE potential temperatures. It should be okay if it doesn't
824
+ # (ACE should just get NaNs there) this is just to see how often it happens
825
+ min_geos_pt = np.nanmin(geos_data_at_ace_times['PT'][i, pt_incr:])
826
+ min_ace_pt = np.nanmin(ace_theta[i, :])
827
+ if min_geos_pt > min_ace_pt:
828
+ logger.info('Note that GEOS PT starts increasing monotonically above the bottom ACE level (profile time: {date}, ACE bottom: {ace:.2f}, GEOS bottom: {geos:.2f})'
829
+ .format(date=ace_dates[i], ace=min_ace_pt, geos=min_geos_pt))
830
+
831
+ # This is one case where I think we do not want to extrapolate because the ACE data spans much greater
832
+ # vertical extent than the GEOS data, so it can't be reliable
833
+ ace_profiles[i, :] = np.interp(ace_theta[i, :], geos_data_at_ace_times['PT'][i, pt_incr:], geos_data[i, pt_incr:],
834
+ left=np.nan, right=np.nan)
835
+ geos_data_on_ace_levels[varname] = ace_profiles
836
+
837
+ # Use the equivalent latitude and potential temperature profiles to look up CLAMS ages #
838
+ ace_age = np.full_like(ace_theta, np.nan)
839
+ for i in range(ace_lon.size):
840
+ if use_geos_theta_for_age:
841
+ theta_prof = geos_data_on_ace_levels['PT'][i, :]
842
+ else:
843
+ theta_prof = ace_theta[i, :]
844
+
845
+ eqlat_prof = geos_data_on_ace_levels['EL'][i, :]
846
+ ace_doy = mod_utils.clams_day_of_year(ace_dates[i])
847
+
848
+ ace_age[i, :] = tccon_priors.get_clams_age(theta_prof, eqlat_prof, ace_doy)
849
+
850
+ geos_data_on_ace_levels['age'] = ace_age
851
+ return geos_data_on_ace_levels
852
+ except Exception as err:
853
+ msg = f'Problem occurred in ACE profile group for dates {date_str} (GEOS file date {geos_dates[0]}): {err}'
854
+ raise err.__class__(msg)
855
+
856
+
857
+ def _bin_ace_to_geos_times(ace_dates):
858
+ """
859
+ Group ACE profiles by which GEOS FP files surround them
860
+
861
+ :param ace_dates: an array of ACE profile datetimes
862
+ :type ace_dates: array of datetimes
863
+
864
+ :return: a dictionary where the keys will be Pandas timestamps representing the time of the GEOS file preceding the
865
+ profiles in that group and the values are the indices of the profiles in that group as a numpy array.
866
+ :rtype: dict
867
+ """
868
+ def to_3hr(d):
869
+ return dt.datetime(d.year, d.month, d.day, to_3(d.hour))
870
+
871
+ def to_3(v):
872
+ return int(3*np.floor(v/3))
873
+
874
+ first_time = ace_dates.min()
875
+ last_time = ace_dates.max()
876
+
877
+ first_time = to_3hr(first_time)
878
+ last_time = to_3hr(last_time) + dt.timedelta(hours=3)
879
+
880
+ # Create the bins as floats so we can use numpy.digitize
881
+ ace_datenums = np.array([sat_utils.datetime2datenum(d) for d in ace_dates])
882
+ bin_edges = pd.date_range(first_time, last_time, freq='3H')
883
+ bin_edge_datenums = np.array([sat_utils.datetime2datenum(d) for d in bin_edges])
884
+
885
+ inds = np.digitize(ace_datenums, bin_edge_datenums) - 1
886
+ date_groups = OrderedDict()
887
+ for i in np.unique(inds):
888
+ date_groups[bin_edges[i]] = np.flatnonzero(inds == i)
889
+
890
+ return date_groups
891
+
892
+
893
+ def _interp_geos_vars_to_ace_lat_lon(geos_file_path, geos_vars, ace_lon, ace_lat):
894
+ def interp_prof_helper(data, glat, glon, alat, alon, interp_inds):
895
+ profs_out = np.full([alat.size, data.shape[0]], np.nan)
896
+ for i, data_layer in enumerate(data):
897
+ # Reshape to nprofs-by-nlevels to be consistent with ACE
898
+ profs_out[:, i] = mm.lat_lon_interp(data_layer, glat, glon, alat, alon, interp_inds)
899
+
900
+ return profs_out
901
+
902
+ geos_data = dict()
903
+ geos_interp_inds = []
904
+ with ncdf.Dataset(geos_file_path) as nh:
905
+ geos_lon_half_res = 0.5 * float(nh.LongitudeResolution)
906
+ geos_lat_half_res = 0.5 * float(nh.LatitudeResolution)
907
+
908
+ for varname, scale in geos_vars.items():
909
+ # convert to 3D - remove singleton time dimension
910
+ geos_data[varname] = nh.variables[varname][0].filled(np.nan) * scale
911
+
912
+ for lon, lat in zip(ace_lon, ace_lat):
913
+ geos_interp_inds.append(mm.querry_indices(nh, lat, lon, geos_lat_half_res, geos_lon_half_res))
914
+
915
+ geos_lat = nh.variables['lat'][:].filled(np.nan)
916
+ geos_lon = nh.variables['lon'][:].filled(np.nan)
917
+
918
+ for varname, array in geos_data.items():
919
+ geos_data[varname] = interp_prof_helper(array, geos_lat, geos_lon, ace_lat, ace_lon, geos_interp_inds)
920
+
921
+ return geos_data
922
+
923
+
924
+ def _interp_geos_vars_to_ace_times(ace_datetimes, geos_datetimes, geos_vars):
925
+ interped_vars = dict()
926
+ for ace_dt in ace_datetimes:
927
+ weight = sat_utils.time_weight_from_datetime(ace_dt, geos_datetimes[0], geos_datetimes[1])
928
+ for k in geos_vars[0].keys():
929
+ interped_vars[k] = weight * geos_vars[0][k] + (1 - weight) * geos_vars[1][k]
930
+
931
+ return interped_vars
932
+
933
+
934
+ def save_ace_age_file(ace_out_file, vars_to_copy, orig_ace_file, eqlat, age):
935
+ def get_var_attrs(nh, varname):
936
+ return {k: nh.variables[varname].getncattr(k) for k in nh.variables[varname].ncattrs()}
937
+
938
+ with ncdf.Dataset(orig_ace_file, 'r') as nhread, ncdf.Dataset(ace_out_file, 'w') as nhwrite:
939
+ # Copy all the dimensions
940
+ for dimname in nhread.dimensions.keys():
941
+ try:
942
+ attributes = get_var_attrs(nhread, dimname)
943
+ ioutils.make_ncdim_helper(nhwrite, dimname, nhread.variables[dimname][:], **attributes)
944
+ except KeyError:
945
+ # Sometimes we get a dimension with no corresponding variable
946
+ nhwrite.createDimension(dimname, nhread.dimensions[dimname].size)
947
+
948
+ # Copy all the requested variables
949
+ for varname in vars_to_copy:
950
+ attributes = get_var_attrs(nhread, varname)
951
+ dims = nhread.variables[varname].dimensions
952
+ ioutils.make_ncvar_helper(nhwrite, varname, nhread.variables[varname][:], dims, **attributes)
953
+
954
+ # Add the new variables
955
+ dims = ('index', 'altitude')
956
+ ioutils.make_ncvar_helper(nhwrite, 'eqlat', eqlat, dims, long_name='equivalent_latitude', units='degrees_north',
957
+ description='Equivalent latitude computed from potential vorticity')
958
+ ioutils.make_ncvar_helper(nhwrite, 'age', age, dims, long_name='clams_age', units='years',
959
+ description='Time in years since the air entered the stratosphere')
960
+
961
+ ioutils.add_creation_info(nhwrite, creation_note='ace_fts_analysis.generate_ace_age_file')
962
+ nhwrite.setncattr('template_ace_file', orig_ace_file)
963
+ ioutils.add_dependent_file_hash(nhwrite, 'template_ace_sha1', orig_ace_file)
964
+
965
+
966
+ def strat_co(ch4, T, P, age, co_0, oh=2.5e5, gamma=3):
967
+ R = 8.314 * 100 ** 3 / 100 / 6.626e23
968
+
969
+ def ndens_air(T, P):
970
+ return P / (R * T)
971
+
972
+ def kCO_OH(nair):
973
+ return 1.57e-13 + 3.54e-33 * nair
974
+
975
+ def kCH4_OH(T):
976
+ return 2.8e-14 * T ** 0.667 * np.exp(-1575 / T)
977
+
978
+ ndens = ndens_air(T, P)
979
+ age = age * 365.25 * 24 * 3600
980
+ ss = gamma * kCH4_OH(T) * ch4 / kCO_OH(ndens)
981
+ bc = co_0 * np.exp(-kCO_OH(ndens) * oh * age)
982
+ return ss + bc
983
+
984
+
985
+ def make_cmam_excess_co_lut(save_file, cmam_file):
986
+
987
+ # First we need to rearrange the CMAM data so that year and month are organized separately.
988
+ with xr.open_dataset(cmam_file) as ds:
989
+ month_slices = [ds['vmrco'][i::12] for i in range(12)]
990
+ for i, month in enumerate(month_slices):
991
+ doy = mod_utils.day_of_year(pd.Timestamp(2005, i+1, 15))
992
+ mdim = OrderedDict([('doy', [doy])])
993
+ month = month.expand_dims(mdim, axis=1)
994
+ month.coords['time'] = [t.item().year for t in month.coords['time']]
995
+ month_slices[i] = month
996
+
997
+ # Duplicate the first and last months on the opposite ends to make periodic interpolation possible
998
+ first_month = month_slices[0].copy()
999
+ first_month.coords['doy'] = first_month.coords['doy'] + mod_utils.days_per_year
1000
+
1001
+ last_month = month_slices[-1].copy()
1002
+ last_month.coords['doy'] = last_month.coords['doy'] - mod_utils.days_per_year
1003
+
1004
+ month_slices.append(first_month)
1005
+ month_slices.insert(0, last_month)
1006
+
1007
+ co_monthly = xr.concat(month_slices, dim='doy')
1008
+
1009
+ # Limit it to after 2000 - there is a trend in CO but it looks like it more or less levels off after 2000.
1010
+ co_attrs = co_monthly.attrs
1011
+ co_monthly = co_monthly.sel(time=slice(2000, None))
1012
+
1013
+ # Then we just need to average in time and longitude to get monthly pressure-latitude slices.
1014
+ co_monthly = co_monthly.mean(dim='time').mean(dim='lon')
1015
+ co_monthly.attrs = co_attrs.copy()
1016
+ # will convert to ppb later, more convenient to leave in mole fraction for now because we need to calculate number
1017
+ # density too.
1018
+
1019
+ # Next we need to calculate the number density. Since we only have pressure from the model, we'll get temperature
1020
+ # from the US standard atmosphere.
1021
+ co_monthly.coords['plev'] /= 100 # convert Pa to hPa
1022
+ co_monthly.coords['plev'].attrs['units'] = 'hPa'
1023
+
1024
+ p = co_monthly.coords['plev']
1025
+ t, z = mod_utils.get_ussa_for_pres(p)
1026
+ t = xr.DataArray(t, coords=p.coords)
1027
+ z = xr.DataArray(z, coords=p.coords)
1028
+
1029
+ t.attrs['units'] = 'K'
1030
+ t.attrs['standard_name'] = 'absolute_temperature'
1031
+ t.attrs['long_name'] = 'temperature'
1032
+
1033
+ z.attrs['units'] = 'km'
1034
+ z.attrs['standard_name'] = 'altitude_above_sea_level'
1035
+ z.attrs['long_name'] = 'altitude'
1036
+
1037
+ nair = mod_utils.number_density_air(p, t)
1038
+ nair.attrs['units'] = 'molec. cm^{-3}'
1039
+ nair.attrs['standard_name'] = 'dry_number_density'
1040
+ nair.attrs['long_name'] = 'number density of air'
1041
+
1042
+ co_monthly_nd = co_monthly * nair
1043
+ co_monthly_nd.attrs['units'] = 'molec. cm^{-3}'
1044
+ co_monthly_nd.attrs['standard_name'] = 'co_number_density'
1045
+ co_monthly_nd.attrs['long_name'] = 'CO number density'
1046
+
1047
+ co_monthly *= 1e9
1048
+ co_monthly.attrs['units'] = 'ppb'
1049
+
1050
+ full_ds = xr.Dataset({'co': co_monthly, 'co_nd': co_monthly_nd, 'temperature': t, 'altitude': z, 'nair': nair})
1051
+ full_ds.attrs['cmam_file'] = cmam_file if not os.path.islink(cmam_file) else os.readlink(cmam_file)
1052
+ full_ds.attrs['cmam_sha1'] = ioutils.make_dependent_file_hash(cmam_file)
1053
+ full_ds.attrs['history'] = ioutils.make_creation_info('ace_fts_analysis.make_cmam_excess_co_lut')
1054
+ full_ds.to_netcdf(save_file)
1055
+
1056
+
1057
+ def make_excess_co_lut(save_file, ace_co_file, ace_ch4_file, ace_age_file, lat_type='eq', min_req_pts=10, smoothing_window_width=5):
1058
+ R = 8.314 * 100 ** 3 / 100 / 6.626e23
1059
+ flag_bits = {'clip': 2 ** 0, 'toofew': 2 ** 1, 'filled': 2 ** 2, 'ussa_any': 2**3, 'ussa_half': 2**4,
1060
+ 'lowval': 2**5}
1061
+ flag_units = 'Bit meanings (least significant first): 1 = value <0 clipped, 2 = fewer than {} points, ' \
1062
+ '3 = NaN filled, 4 = At least one point used USSA P/T, 5 = >50% points used USSA P/T, ' \
1063
+ '6 = value below rolling mean - 1 sigma filled'.format(min_req_pts)
1064
+
1065
+ def ndens_air(T, P):
1066
+ return P / (R * T)
1067
+
1068
+ def kCO_OH(nair):
1069
+ return 1.57e-13 + 3.54e-33 * nair
1070
+
1071
+ def kCH4_OH(T):
1072
+ return 2.8e-14 * T ** 0.667 * np.exp(-1575 / T)
1073
+
1074
+ def strat_co(ch4, T, P, age, co_0, oh=2.5e5, gamma=3):
1075
+ ndens = ndens_air(T, P)
1076
+ age = age * 365.25 * 24 * 3600
1077
+ ss = gamma * kCH4_OH(T) * ch4 / kCO_OH(ndens)
1078
+ bc = co_0 * np.exp(-kCO_OH(ndens) * oh * age)
1079
+ return ss + bc
1080
+
1081
+ def fill_nans(ds, flags, primary_keys, dim_interp_order):
1082
+ # We want to interpolate/extrapolate along theta first. I tried originally doing periodic interpolation along
1083
+ # the day-of-year axis, but there's not enough data in the northen hemisphere polar summer to get the decrease
1084
+ # right. However, there are some columns that have 0 or 1 non-NaN values which the interpolator then chokes
1085
+ # on, so we need to find those columns and replace the NaNs with a fill value temporarily to avoid trying
1086
+ # to handle them here.
1087
+ was_nans = np.zeros(ds[primary_keys[0]].shape, dtype=np.bool_)
1088
+ for k in primary_keys:
1089
+ was_nans |= np.isnan(ds[k].data)
1090
+
1091
+ all_nans = dict()
1092
+ for dim, do_extrap in dim_interp_order.items():
1093
+ logger.debug('Filling NaNs along'.format(dim))
1094
+
1095
+ fill_val = 'extrapolate' if do_extrap else np.nan
1096
+
1097
+ for key, variable in ds.items():
1098
+ var_not_nans = ~np.isnan(variable.data)
1099
+
1100
+ # DataArray's sum function doesn't work with keepdims in v0.12.1. This is fixed in v0.12.2 which doesn't
1101
+ # seem to yet be available via conda
1102
+ dim_ind = variable.dims.index(dim)
1103
+ this_most_nans = var_not_nans.sum(axis=dim_ind, keepdims=True) < 2
1104
+ this_most_nans = np.broadcast_to(this_most_nans, variable.shape)
1105
+ # only replace NaNs in lines with 0 or 1 non-NaN values. Leave non-NaN values in those columns alone,
1106
+ # and leave NaNs elsewhere alone.
1107
+ this_most_nans = this_most_nans & ~var_not_nans
1108
+ variable.data[this_most_nans] = -999
1109
+ all_nans[key] = this_most_nans
1110
+
1111
+ ds = ds.interpolate_na(dim=dim, method='linear', fill_value=fill_val)
1112
+
1113
+ for key, this_most_nans in all_nans.items():
1114
+ ds[key].data[this_most_nans] = np.nan
1115
+
1116
+ nans_filled = np.zeros_like(was_nans)
1117
+ for k in primary_keys:
1118
+ nans_filled |= (was_nans & ~np.isnan(ds[k].data))
1119
+
1120
+ flags[nans_filled] = np.bitwise_or(flags[nans_filled], flag_bits['filled'])
1121
+
1122
+ return ds, flags
1123
+
1124
+ def bin_co(df_dict, all_bins, all_coords, bin_names, primary_key):
1125
+ bin_cols = tuple(df_dict.keys())
1126
+ coord_cols = []
1127
+ total_bins = 1
1128
+ for i, (coord, bin) in enumerate(zip(all_coords, all_bins)):
1129
+ total_bins *= (bin.size - 1)
1130
+ inds = np.digitize(coord.flatten(), bin) - 1
1131
+ colname = 'ind{}'.format(i)
1132
+ df_dict[colname] = inds
1133
+ coord_cols.append(colname)
1134
+
1135
+ df = pd.DataFrame(df_dict)
1136
+ all_bin_centers = [0.5 * (b[:-1] + b[1:]) for b in all_bins]
1137
+
1138
+ proto_array = np.full([np.size(b) - 1 for b in all_bins],
1139
+ np.nan if np.issubdtype(excess_co.dtype, np.floating) else 0)
1140
+ proto_array = xr.DataArray(proto_array, coords=all_bin_centers, dims=bin_names)
1141
+ means = xr.Dataset({k: proto_array.copy() for k in bin_cols})
1142
+ stds = xr.Dataset({k: proto_array.copy() for k in bin_cols})
1143
+
1144
+ flags = np.zeros(proto_array.shape, dtype=int)
1145
+ counts = np.zeros(proto_array.shape, dtype=int)
1146
+
1147
+ pbar = mod_utils.ProgressBar(total_bins, prefix='Binning', style='counter')
1148
+ iind = 0
1149
+ for inds, grp in df.groupby(coord_cols):
1150
+ pbar.print_bar(iind)
1151
+ iind += 1
1152
+
1153
+ if any(i < 0 for i in inds):
1154
+ # negative index indicates that the value is below the minimum bin so we skip it
1155
+ continue
1156
+ try:
1157
+ subdata = grp[primary_key]
1158
+ counts[inds] = (~subdata.isna()).sum()
1159
+ if subdata.size < min_req_pts:
1160
+ flags[inds] = np.bitwise_or(flags[inds], flag_bits['toofew'])
1161
+ continue
1162
+
1163
+ for k in bin_cols:
1164
+ subdata = grp[k]
1165
+ means[k][inds] = np.nanmean(subdata)
1166
+ stds[k][inds] = np.nanstd(subdata)
1167
+
1168
+ except IndexError:
1169
+ # inds will be too large if any of the values is outside the right edge of the last bin, so just skip
1170
+ # those since we don't want to bin them. We caught any outside the left edge of the bin checking if any
1171
+ # are < 0
1172
+ continue
1173
+
1174
+ pbar.finish()
1175
+
1176
+ return means, stds, flags, counts, all_bin_centers
1177
+
1178
+ def replace_low_values(darray, dim, flags, remove_neg=True, extra_mask=None):
1179
+ darray = darray.copy()
1180
+ periodic = dim == 'doy'
1181
+ # Concatenate extra slices to allow periodic interpolation if operating along DOY
1182
+ if periodic:
1183
+ ex_width = int(np.ceil(smoothing_window_width/2))
1184
+ dim_max = darray.coords[dim].max()
1185
+ darray_start = darray.isel(**{dim: slice(None, ex_width)})
1186
+ darray_start.coords[dim] = darray_start.coords[dim] + dim_max
1187
+ darray_end = darray.isel(**{dim: slice(-ex_width, None)})
1188
+ darray_end.coords[dim] = darray_end.coords[dim] - dim_max
1189
+ darray = xr.concat([darray_end, darray, darray_start], dim=dim)
1190
+
1191
+ if remove_neg:
1192
+ darray.data[darray.data < 0] = np.nan
1193
+ kwargs = {dim: smoothing_window_width, 'center': True}
1194
+ mean = darray.rolling(**kwargs).mean()
1195
+ std = darray.rolling(**kwargs).std()
1196
+ xx = darray < (mean - std)
1197
+ if extra_mask is not None:
1198
+ xx = xx | extra_mask
1199
+ darray.data[xx.data] = np.nan
1200
+
1201
+ darray = darray.interpolate_na(dim=dim)
1202
+ if periodic:
1203
+ s = slice(ex_width, -ex_width)
1204
+ darray = darray.isel(**{dim: s})
1205
+ xx = xx.isel(**{dim: s})
1206
+ flags[xx.data] = np.bitwise_or(flags[xx.data], flag_bits['lowval'])
1207
+ return darray, flags
1208
+
1209
+ #################
1210
+ # Load ACE data #
1211
+ #################
1212
+
1213
+ with ncdf.Dataset(ace_age_file) as ds:
1214
+ ace_dates = read_ace_date(ds)
1215
+ ace_alt = read_ace_var(ds, 'altitude', None)
1216
+ qflags = read_ace_var(ds, 'quality_flag', None)
1217
+
1218
+ if lat_type == 'geog':
1219
+ ace_lat = read_ace_var(ds, 'latitude', None)
1220
+ ace_lat = np.broadcast_to(ace_lat.reshape(-1, 1), qflags.shape)
1221
+ elif lat_type == 'eq':
1222
+ ace_lat = read_ace_var(ds, 'eqlat', None)
1223
+ # Equivalent latitude may not be defined on all levels. Fill in any internal levels with linear
1224
+ # interpolation then extends the profile assuming that the equivalent latitude is constant above the top
1225
+ # level.
1226
+ ace_lat = xr.DataArray(ace_lat, coords=[np.arange(ace_lat.shape[0]), ace_alt], dims=['index', 'alt'])
1227
+ ace_lat = ace_lat.interpolate_na(dim='alt', method='linear').interpolate_na(dim='alt', method='nearest', fill_value='extrapolate')
1228
+ ace_lat = ace_lat.data
1229
+ else:
1230
+ raise ValueError('lat must be "geog" or "eq"')
1231
+
1232
+ ace_t = read_ace_var(ds, 'temperature', qflags)
1233
+ ace_p = read_ace_var(ds, 'pressure', qflags) * 1013.25 # atm -> hPa
1234
+ ace_pt = read_ace_theta(ds, qflags=qflags)
1235
+ ace_age = read_ace_var(ds, 'age', qflags)
1236
+
1237
+ with ncdf.Dataset(ace_ch4_file) as ds:
1238
+ qflags = read_ace_var(ds, 'quality_flag', None)
1239
+ ace_ch4 = read_ace_var(ds, 'CH4', qflags)
1240
+
1241
+ with ncdf.Dataset(ace_co_file) as ds:
1242
+ qflags = read_ace_var(ds, 'quality_flag', None)
1243
+ all_ace_co = read_ace_var(ds, 'CO', qflags)
1244
+
1245
+ ace_doys = np.array([mod_utils.day_of_year(d) for d in ace_dates])
1246
+ ace_doys = np.broadcast_to(ace_doys.reshape(-1, 1), all_ace_co.shape)
1247
+
1248
+ # Create versions of T and P filled in by the US standard atmosphere for the mesospheric CO profile table.
1249
+ xx_ussa = np.isnan(ace_t) | np.isnan(ace_p)
1250
+ ace_alt_full = np.broadcast_to(ace_alt.reshape(1, -1), ace_t.shape)
1251
+ ace_t_ussa = ace_t.copy()
1252
+ ace_p_ussa = ace_p.copy()
1253
+ ace_t_ussa[xx_ussa], ace_p_ussa[xx_ussa] = mod_utils.get_ussa_for_alts(ace_alt_full[xx_ussa])
1254
+
1255
+ co_calc = strat_co(ace_ch4, ace_t, ace_p, ace_age, 50.0e-9)
1256
+ excess_co = (all_ace_co - co_calc)*1e9 # calculate excess and convert to ppb
1257
+ excess_notnans = ~np.isnan(excess_co)
1258
+ excess_co = excess_co[excess_notnans]
1259
+
1260
+ air_nd = mod_utils.number_density_air(ace_p_ussa, ace_t_ussa)
1261
+ ace_co_nd = all_ace_co * air_nd
1262
+ not_all_nans = np.any(~np.isnan(ace_co_nd), axis=0)
1263
+ air_nd = air_nd[:, not_all_nans]
1264
+ ace_co_nd = ace_co_nd[:, not_all_nans]
1265
+ ace_p_ussa = ace_p_ussa[:, not_all_nans]
1266
+ ace_t_ussa = ace_t_ussa[:, not_all_nans]
1267
+ xx_ussa = xx_ussa[:, not_all_nans]
1268
+ ace_alt_full = ace_alt_full[:, not_all_nans]
1269
+
1270
+ meso_notnans = ~np.isnan(ace_co_nd)
1271
+ ace_co_nd = ace_co_nd[meso_notnans]
1272
+ ace_p_ussa = ace_p_ussa[meso_notnans]
1273
+ ace_t_ussa = ace_t_ussa[meso_notnans]
1274
+ air_nd = air_nd[meso_notnans]
1275
+ xx_ussa = xx_ussa[meso_notnans]
1276
+
1277
+ lat_bins = np.arange(-90, 91, 10)
1278
+ doy_bins = np.arange(0, 370, 5)
1279
+ doy_bins[-1] = 366 # account for leap years
1280
+ theta_bins = np.arange(350, 4150, 100)
1281
+ alt_bins = np.arange(ace_alt[not_all_nans].min(), ace_alt[not_all_nans].max()+5, 5.)
1282
+
1283
+ excess_bins = [lat_bins, doy_bins, theta_bins]
1284
+ excess_coords = [ace_lat[excess_notnans], ace_doys[excess_notnans], ace_pt[excess_notnans]]
1285
+ excess_bin_names = ['lat', 'doy', 'theta']
1286
+ excess_bin_units = ['degrees_north', 'day of year (0-based)', 'K']
1287
+
1288
+ meso_bins = [lat_bins, doy_bins, alt_bins]
1289
+ meso_coords = [ace_lat[:, not_all_nans][meso_notnans], ace_doys[:, not_all_nans][meso_notnans], ace_alt_full[meso_notnans]]
1290
+ meso_bin_names = ['lat', 'doy', 'altitude']
1291
+ meso_bin_units = ['degrees_north', 'doy of year (0-based)', 'km']
1292
+
1293
+ ####################################################
1294
+ # Use pandas groupby() to bin the data efficiently #
1295
+ ####################################################
1296
+
1297
+ excess_means, excess_stds, excess_flags, excess_counts, excess_bin_centers = bin_co(
1298
+ {'excess_co': excess_co}, all_bins=excess_bins, all_coords=excess_coords, bin_names=excess_bin_names,
1299
+ primary_key='excess_co'
1300
+ )
1301
+ excess_means['excess_co'].attrs['unit'] = 'ppb'
1302
+ excess_means['excess_co'].attrs['description'] = 'Excess CO in ACE-FTS data beyond that predicted by a kinetic ' \
1303
+ 'model of the CO chemistry in the stratosphere.'
1304
+ excess_stds['excess_co'].attrs['unit'] = 'ppb'
1305
+ excess_stds['excess_co'].attrs['description'] = 'Standard deviation of the excess CO in this bin.'
1306
+
1307
+ meso_means, meso_stds, meso_flags, meso_counts, meso_bin_centers = bin_co(
1308
+ {'co_nd': ace_co_nd, 'air_nd': air_nd, 'pressure': ace_p_ussa, 'temperature': ace_t_ussa, 'is_ussa': xx_ussa},
1309
+ all_bins=meso_bins, all_coords=meso_coords, bin_names=meso_bin_names, primary_key='co_nd'
1310
+ )
1311
+
1312
+ inds = meso_means['is_ussa'].data > 0
1313
+ meso_flags[inds] = np.bitwise_or(meso_flags[inds], flag_bits['ussa_any'])
1314
+ inds = meso_means['is_ussa'].data > 0.5
1315
+ meso_flags[inds] = np.bitwise_or(meso_flags[inds], flag_bits['ussa_half'])
1316
+
1317
+ meso_means = meso_means.drop('is_ussa')
1318
+ meso_stds = meso_stds.drop('is_ussa')
1319
+
1320
+ meso_means['co_nd'].attrs['unit'] = 'molec cm^{-3}'
1321
+ meso_means['co_nd'].attrs['description'] = 'ACE CO in number density. ACE T and P extended using the US standard ' \
1322
+ 'atmosphere as needed.'
1323
+ meso_means['air_nd'].attrs['unit'] = 'molec cm^{-3}'
1324
+ meso_means['air_nd'].attrs['description'] = 'Number density of air calculated using the ideal gas law with ACE T ' \
1325
+ 'and P extended using the US standard atmosphere as needed'
1326
+ meso_means['pressure'].attrs['unit'] = 'hPa'
1327
+ meso_means['pressure'].attrs['description'] = 'ACE pressure extended using the US standard atmosphere as needed'
1328
+ meso_means['temperature'].attrs['unit'] = 'K'
1329
+ meso_means['temperature'].attrs['description'] = 'ACE temperatures extended using the US standard atmosphere as needed'
1330
+
1331
+ meso_stds['co_nd'].attrs['unit'] = 'molec cm^{-3}'
1332
+ meso_stds['co_nd'].attrs['description'] = 'Standard deviation of ACE CO in number density in the bins.'
1333
+ meso_stds['air_nd'].attrs['unit'] = 'molec cm^{-3}'
1334
+ meso_stds['air_nd'].attrs['description'] = 'Standard deviation of number density of air in the bins'
1335
+ meso_stds['pressure'].attrs['unit'] = 'hPa'
1336
+ meso_stds['pressure'].attrs['description'] = 'Standard deviation of ACE pressure in the bins.'
1337
+ meso_stds['temperature'].attrs['unit'] = 'K'
1338
+ meso_stds['temperature'].attrs['description'] = 'Standard deviation of ACE temperatures in the bins'
1339
+
1340
+ all_bin_edges = excess_bins + meso_bins[-1:]
1341
+ all_bin_centers = excess_bin_centers + meso_bin_centers[-1:]
1342
+ all_bin_names = excess_bin_names + meso_bin_names[-1:]
1343
+ all_bin_units = excess_bin_units + meso_bin_units[-1:]
1344
+
1345
+ #######################################################################################
1346
+ # Fill in NaNs and clip to positive values. Track these actions in the flags variable #
1347
+ #######################################################################################
1348
+ inds = excess_means['excess_co'].data < 0
1349
+ excess_flags[inds] = np.bitwise_or(excess_flags[inds], flag_bits['clip'])
1350
+ excess_means['excess_co'] = excess_means['excess_co'].clip(0)
1351
+
1352
+ excess_interp_order = OrderedDict([('theta', True), ('lat', True), ('doy', False)])
1353
+ excess_means, excess_flags = fill_nans(excess_means, excess_flags, ['excess_co'], excess_interp_order)
1354
+ meso_interp_order = OrderedDict([('doy', False), ('lat', True), ('altitude', True)])
1355
+ meso_means, meso_flags = fill_nans(meso_means, meso_flags, ['co_nd', 'air_nd', 'pressure', 'temperature'],
1356
+ meso_interp_order)
1357
+
1358
+ excess_means['excess_co'], excess_flags = replace_low_values(excess_means['excess_co'], 'doy', excess_flags)
1359
+ meso_means['co_nd'], meso_flags = replace_low_values(meso_means['co_nd'], 'doy', meso_flags)
1360
+
1361
+ # Add the count variables
1362
+ excess_means['excess_co_counts'] = xr.DataArray(excess_counts, coords=excess_means['excess_co'].coords)
1363
+ excess_means['excess_co_counts'].attrs['unit'] = '#'
1364
+ excess_means['excess_co_counts'].attrs['description'] = 'Number of ACE points contributing to the bin'
1365
+ meso_means['co_nd_counts'] = xr.DataArray(meso_counts, coords=meso_means['co_nd'].coords)
1366
+ meso_means['co_nd_counts'].attrs['unit'] = '#'
1367
+ meso_means['co_nd_counts'].attrs['description'] = 'Number of ACE points contributing to the bin'
1368
+
1369
+ _save_excess_co_lut(save_file=save_file,
1370
+ theta_binned_means=excess_means, theta_binned_stds=excess_stds, theta_binned_flags=excess_flags,
1371
+ alt_binned_means=meso_means, alt_binned_stds=meso_stds, alt_binned_flags=meso_flags,
1372
+ bin_edges=all_bin_edges, bin_centers=all_bin_centers, bin_names=all_bin_names,
1373
+ bin_units=all_bin_units, theta_binned_dims=excess_bin_names, alt_binned_dims=meso_bin_names,
1374
+ flag_description=flag_units, lat_type=lat_type,
1375
+ ace_co_file=ace_co_file, ace_ch4_file=ace_ch4_file, ace_age_file=ace_age_file)
1376
+
1377
+
1378
+ def _save_excess_co_lut(save_file, theta_binned_means, theta_binned_stds, theta_binned_flags,
1379
+ alt_binned_means, alt_binned_stds, alt_binned_flags,
1380
+ bin_edges, bin_centers, bin_names, bin_units,
1381
+ flag_description, theta_binned_dims, alt_binned_dims, lat_type,
1382
+ ace_co_file, ace_ch4_file, ace_age_file):
1383
+
1384
+ with ncdf.Dataset(save_file, 'w') as nh:
1385
+
1386
+ dims = []
1387
+ for name, centers, edges, units in zip(bin_names, bin_centers, bin_edges, bin_units):
1388
+ this_dim = ioutils.make_ncdim_helper(nh, name, centers, unit=units)
1389
+ dims.append(this_dim)
1390
+ ioutils.make_ncdim_helper(nh, name + '_bin_edges', edges, unit=units)
1391
+
1392
+ for k in theta_binned_means.keys():
1393
+ data = theta_binned_means[k]
1394
+ ioutils.make_ncvar_helper(nh, k, data.data, dims=theta_binned_dims,
1395
+ flag_field='theta_binned_flags', **data.attrs)
1396
+ if k in theta_binned_stds:
1397
+ std = theta_binned_stds[k]
1398
+ ioutils.make_ncvar_helper(nh, k+'_std', std.data, dims=theta_binned_dims,
1399
+ flag_field='theta_binned_flags', **std.attrs)
1400
+ ioutils.make_ncvar_helper(nh, 'theta_binned_flags', theta_binned_flags, dims=theta_binned_dims,
1401
+ unit='Bit array flag', description=flag_description)
1402
+
1403
+ for k in alt_binned_means.keys():
1404
+ data = alt_binned_means[k]
1405
+ ioutils.make_ncvar_helper(nh, k, data.data, dims=alt_binned_dims,
1406
+ flag_field='alt_binned_flags', **data.attrs)
1407
+ if k in alt_binned_stds:
1408
+ std = alt_binned_stds[k]
1409
+ ioutils.make_ncvar_helper(nh, k+'_std', std.data, dims=alt_binned_dims,
1410
+ flag_field='alt_binned_flags', **std.attrs)
1411
+ ioutils.make_ncvar_helper(nh, 'alt_binned_flags', alt_binned_flags, dims=alt_binned_dims,
1412
+ unit='Bit array flag', description=flag_description)
1413
+
1414
+ ioutils.add_dependent_file_hash(nh, 'ace_co_file', ace_co_file)
1415
+ ioutils.add_dependent_file_hash(nh, 'ace_ch4_file', ace_ch4_file)
1416
+ ioutils.add_dependent_file_hash(nh, 'ace_age_file', ace_age_file)
1417
+ ioutils.add_creation_info(nh, 'ace_fts_analysis.create_excess_co_lut')
1418
+ nh.setncattr('latitude_type', 'Binned by {} latitude'.format(lat_type))