ginput 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ginput/__init__.py +4 -0
- ginput/common_utils/__init__.py +0 -0
- ginput/common_utils/ggg_logging.py +62 -0
- ginput/common_utils/ioutils.py +304 -0
- ginput/common_utils/mod_constants.py +24 -0
- ginput/common_utils/mod_utils.py +2663 -0
- ginput/common_utils/readers.py +477 -0
- ginput/common_utils/run_utils.py +57 -0
- ginput/common_utils/sat_utils.py +67 -0
- ginput/common_utils/versioning.py +251 -0
- ginput/common_utils/writers.py +377 -0
- ginput/data/.gitignore +1 -0
- ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
- ginput/data/ML_monthly_obs_ch4.txt +498 -0
- ginput/data/ML_monthly_obs_co.txt +412 -0
- ginput/data/ML_monthly_obs_co2.txt +597 -0
- ginput/data/ML_monthly_obs_n2o.txt +330 -0
- ginput/data/SMO_monthly_obs_ch4.txt +499 -0
- ginput/data/SMO_monthly_obs_co.txt +416 -0
- ginput/data/SMO_monthly_obs_co2.txt +613 -0
- ginput/data/SMO_monthly_obs_n2o.txt +330 -0
- ginput/data/ace_fn2o_lut.nc +0 -0
- ginput/data/age_spectra/age.midlat.txt +47 -0
- ginput/data/age_spectra/age.tropics.txt +46 -0
- ginput/data/age_spectra/age.vortex.txt +31 -0
- ginput/data/age_spectra/agespec.midlat.txt +47 -0
- ginput/data/age_spectra/agespec.tropics.txt +46 -0
- ginput/data/age_spectra/agespec.vortex.txt +31 -0
- ginput/data/age_spectra/time.txt +3001 -0
- ginput/data/ch4_hf_slopes.nc +0 -0
- ginput/data/ch4_strat_lut.nc +0 -0
- ginput/data/clams_age_clim_scaled.nc +0 -0
- ginput/data/co2_strat_lut.nc +0 -0
- ginput/data/fo2_data/.gitignore +2 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
- ginput/data/hf_strat_lut.nc +0 -0
- ginput/data/isotopologs.dat +199 -0
- ginput/data/meso_co_lut.nc +0 -0
- ginput/data/n2o_ch4_acefts.nc +0 -0
- ginput/data/n2o_strat_lut.nc +0 -0
- ginput/data/o2_mean_dmf.dat +52 -0
- ginput/data/summer_35N.vmr +182 -0
- ginput/data/washenfelder03_table_s3.txt +83 -0
- ginput/download/__init__.py +0 -0
- ginput/download/download_utils.py +46 -0
- ginput/download/get_GEOS5.py +367 -0
- ginput/download/get_MERRA2.py +177 -0
- ginput/download/get_NOAA_flask_data.py +103 -0
- ginput/download/get_fo2_data.py +140 -0
- ginput/mod_maker/__init__.py +0 -0
- ginput/mod_maker/mod_maker.py +2470 -0
- ginput/mod_maker/slantify.py +450 -0
- ginput/mod_maker/tccon_sites.py +570 -0
- ginput/priors/__init__.py +0 -0
- ginput/priors/acos_interface.py +1376 -0
- ginput/priors/automation.py +313 -0
- ginput/priors/backend_analysis/__init__.py +0 -0
- ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
- ginput/priors/backend_analysis/ace_fts_val.py +452 -0
- ginput/priors/backend_analysis/acos_comparison.py +550 -0
- ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
- ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
- ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
- ginput/priors/backend_analysis/backend_driver.py +90 -0
- ginput/priors/backend_analysis/backend_utils.py +358 -0
- ginput/priors/backend_analysis/clams.py +71 -0
- ginput/priors/backend_analysis/create_test_priors.py +853 -0
- ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
- ginput/priors/backend_analysis/mls_val.py +217 -0
- ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
- ginput/priors/fo2_prep.py +311 -0
- ginput/priors/map_maker.py +193 -0
- ginput/priors/mlo_smo_prep.py +1146 -0
- ginput/priors/tccon_priors.py +3482 -0
- ginput-1.3.0.dist-info/LICENSE.txt +203 -0
- ginput-1.3.0.dist-info/METADATA +125 -0
- ginput-1.3.0.dist-info/RECORD +86 -0
- ginput-1.3.0.dist-info/WHEEL +5 -0
- ginput-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1418 @@
|
|
|
1
|
+
from __future__ import print_function, division, absolute_import
|
|
2
|
+
|
|
3
|
+
from collections import OrderedDict
|
|
4
|
+
import datetime as dt
|
|
5
|
+
from glob import glob
|
|
6
|
+
from itertools import repeat
|
|
7
|
+
from matplotlib import pyplot as plt
|
|
8
|
+
from multiprocessing import Pool
|
|
9
|
+
import netCDF4 as ncdf
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from scipy.optimize import curve_fit
|
|
15
|
+
from statsmodels.robust.robust_linear_model import RLM
|
|
16
|
+
from statsmodels.robust.norms import TukeyBiweight
|
|
17
|
+
import xarray as xr
|
|
18
|
+
|
|
19
|
+
from ...common_utils import mod_utils, ioutils, sat_utils
|
|
20
|
+
from ...common_utils.ggg_logging import logger
|
|
21
|
+
from .. import tccon_priors
|
|
22
|
+
from ...mod_maker import mod_maker as mm
|
|
23
|
+
from .backend_utils import read_ace_var, read_ace_date, read_ace_theta, read_ace_latlon
|
|
24
|
+
|
|
25
|
+
_mydir = os.path.abspath(os.path.dirname(__file__))
|
|
26
|
+
_tccon_top_alt = 70.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _bin_centers(bins):
|
|
30
|
+
return bins[:-1] + np.diff(bins)/2
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
_std_age_bins = np.arange(0.0, 10.0, 0.25)
|
|
34
|
+
_std_age_bin_centers = _bin_centers(_std_age_bins)
|
|
35
|
+
_std_frac_bins = np.arange(0.0, 1.05, 0.05)
|
|
36
|
+
_std_frac_bin_centers = _bin_centers(_std_frac_bins)
|
|
37
|
+
_std_theta_bins = np.concatenate([np.arange(380, 1080, 50), np.arange(1080, 1680, 100), np.arange(1680, 3680, 200)])
|
|
38
|
+
_std_theta_bin_centers = _bin_centers(_std_theta_bins)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def ace_lut_driver(ace_dir, rw_supp_table_file, geos_path, lut_save_dir, overwrite_age_file=True, nprocs=0):
|
|
42
|
+
"""Generate all the ACE-derived look-up tables needed for ginput
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
ace_dir : str
|
|
47
|
+
Directory to the ACE-FTS data. Expects the ACEs files therein to be named "ACE*{specie}.nc", e.g.
|
|
48
|
+
"ACEFTS_L2_v3p6_N2O.nc".
|
|
49
|
+
|
|
50
|
+
rw_supp_table_file : str
|
|
51
|
+
Path to text file containing Table S3.1 from Washenfelder et al. 2003 (doi:10.1029/2003GL017969). This is
|
|
52
|
+
usually "washenfelder03_table_s3.txt" in ginput/data.
|
|
53
|
+
|
|
54
|
+
geos_path : str
|
|
55
|
+
Path to the GEOS-FPIT met directory containing the "Nv" subdirectory with eta-level GEOS met files.
|
|
56
|
+
|
|
57
|
+
lut_save_dir : str
|
|
58
|
+
Path to save the newly generated LUT files to
|
|
59
|
+
|
|
60
|
+
overwrite_age_file : bool
|
|
61
|
+
Whether or not to recreate and overwrite the ACE-FTS "AGE" file if it exists.
|
|
62
|
+
|
|
63
|
+
nprocs : int
|
|
64
|
+
Number of processors to use when generating the ACE AGE file.
|
|
65
|
+
|
|
66
|
+
Returns
|
|
67
|
+
-------
|
|
68
|
+
None
|
|
69
|
+
Writes 3 files (ace_fn2o_lut.nc, n2o_ch4_acefts.nc, and ch4_hf_slopes.nc) to `lut_save_dir` and an ACE "AGE"
|
|
70
|
+
file to `ace_dir` if either that file does not exist or `overwrite_age_file` is `True`.
|
|
71
|
+
|
|
72
|
+
"""
|
|
73
|
+
# Make a dictionary of the available ACE files by species
|
|
74
|
+
ace_files = {re.search(r'[A-Z0-9]+(?=\.nc)', os.path.basename(f)).group(): f for f in glob(os.path.join(ace_dir, 'ACE*.nc'))}
|
|
75
|
+
|
|
76
|
+
if 'AGE' not in ace_files or overwrite_age_file:
|
|
77
|
+
age_file = re.sub(r'P.nc$', 'AGE.nc', ace_files['P'])
|
|
78
|
+
logger.info('Creating ACE-FTS AGE file. Will save to {}. Using {} processors'.format(age_file, nprocs))
|
|
79
|
+
generate_ace_age_file(ace_in_file=ace_files['P'], age_file_out=age_file, geos_path=geos_path,
|
|
80
|
+
use_geos_theta_for_age=False, nprocs=nprocs)
|
|
81
|
+
ace_files['AGE'] = age_file
|
|
82
|
+
|
|
83
|
+
fn2o_save_file = os.path.join(lut_save_dir, 'ace_fn2o_lut.nc')
|
|
84
|
+
logger.info('Creating F(N2O) look up table file. Will save to {}.'.format(fn2o_save_file))
|
|
85
|
+
make_fn2o_lookup_table(ace_age_file=ace_files['AGE'], ace_n2o_file=ace_files['N2O'], lut_save_file=fn2o_save_file)
|
|
86
|
+
|
|
87
|
+
fch4_save_file = os.path.join(lut_save_dir, 'n2o_ch4_acefts.nc')
|
|
88
|
+
logger.info('Creating F(CH4):F(N2O) look up table file. Will save to {}.'.format(fch4_save_file))
|
|
89
|
+
make_fch4_fn2o_lookup_table(ace_age_file=ace_files['AGE'], ace_n2o_file=ace_files['N2O'], ace_ch4_file=ace_files['CH4'],
|
|
90
|
+
lut_save_file=fch4_save_file)
|
|
91
|
+
|
|
92
|
+
hf_slope_save_file = os.path.join(lut_save_dir, 'ch4_hf_slopes.nc')
|
|
93
|
+
logger.info('Creating CH4:HF slope look up table file. Will save to {}'.format(hf_slope_save_file))
|
|
94
|
+
make_hf_ch4_slopes(ace_ch4_file=ace_files['CH4'], ace_hf_file=ace_files['HF'], ace_age_file=ace_files['AGE'],
|
|
95
|
+
washenfelder_supp_table_file=rw_supp_table_file, lut_save_file=hf_slope_save_file,
|
|
96
|
+
clip_values=True)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def make_fn2o_lookup_table(ace_age_file, ace_n2o_file, lut_save_file):
|
|
100
|
+
logger.info('Loading ACE data')
|
|
101
|
+
ace_alt, ace_fn2o, ace_theta, extra_info = calc_fraction_remaining_from_acefts(ace_n2o_file, 'N2O', bc_approach='fit')
|
|
102
|
+
|
|
103
|
+
ace_dates = extra_info['dates']
|
|
104
|
+
ace_doy = np.array([int(mod_utils.clams_day_of_year(date)) for date in ace_dates.flat]).reshape(ace_dates.shape)
|
|
105
|
+
|
|
106
|
+
with ncdf.Dataset(ace_n2o_file, 'r') as nch:
|
|
107
|
+
ace_lat = read_ace_var(nch, 'latitude', None)
|
|
108
|
+
|
|
109
|
+
with ncdf.Dataset(ace_age_file, 'r') as nch:
|
|
110
|
+
ace_age = nch.variables['age'][:].filled(np.nan)
|
|
111
|
+
|
|
112
|
+
logger.info('Binning F(N2O) vs. age and theta')
|
|
113
|
+
# Similar to the F(CH4) vs. F(N2O) approach, we ignore fill values, want only positive F(N2O) values, want to
|
|
114
|
+
# avoid the polar vortex and want to stay within TCCON relevant altitudes.
|
|
115
|
+
#
|
|
116
|
+
# is_vortex expects everything to be the same size, so we need to tile lat and doy
|
|
117
|
+
ace_lat = np.tile(ace_lat[:, np.newaxis], [1, ace_age.shape[1]])
|
|
118
|
+
ace_doy = np.tile(ace_doy[:, np.newaxis], [1, ace_age.shape[1]])
|
|
119
|
+
|
|
120
|
+
xx = ~np.isnan(ace_fn2o) & ~np.isnan(ace_age) & (ace_fn2o >= 0) & \
|
|
121
|
+
~mod_utils.is_vortex(ace_lat, ace_doy, ace_age) & (ace_alt[np.newaxis, :] < _tccon_top_alt)
|
|
122
|
+
|
|
123
|
+
age_bins = _std_age_bins
|
|
124
|
+
age_bin_centers = _bin_centers(age_bins)
|
|
125
|
+
theta_bins = _std_theta_bins
|
|
126
|
+
theta_bin_centers = _bin_centers(theta_bins)
|
|
127
|
+
|
|
128
|
+
# We're not going to worry about outliers initially
|
|
129
|
+
fn2o_means, fn2o_counts, fn2o_overall, theta_overall = _bin_z_vs_xy(ace_age, ace_theta, ace_fn2o, xx, age_bins, theta_bins)
|
|
130
|
+
|
|
131
|
+
fn2o_means[fn2o_means > 1] = 1
|
|
132
|
+
fn2o_overall[fn2o_overall > 1] = 1
|
|
133
|
+
|
|
134
|
+
_save_fn2o_lut(lut_save_file, fn2o_means=fn2o_means, fn2o_counts=fn2o_counts, fn2o_overall=fn2o_overall,
|
|
135
|
+
theta_overall=theta_overall, age_bin_centers=age_bin_centers, age_bin_edges=age_bins,
|
|
136
|
+
theta_bin_centers=theta_bin_centers, theta_bin_edges=theta_bins,
|
|
137
|
+
ace_n2o_file=ace_n2o_file, ace_age_file=ace_age_file)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def make_fch4_fn2o_lookup_table(ace_age_file, ace_n2o_file, ace_ch4_file, lut_save_file):
|
|
141
|
+
logger.info('Reading ACE data')
|
|
142
|
+
ace_alt, ace_fn2o, ace_theta, extra_info = calc_fraction_remaining_from_acefts(ace_n2o_file, 'N2O', bc_approach='fit')
|
|
143
|
+
ace_dates = extra_info['dates']
|
|
144
|
+
ace_doy = np.array([int(mod_utils.clams_day_of_year(date)) for date in ace_dates.flat]).reshape(ace_dates.shape)
|
|
145
|
+
|
|
146
|
+
_, ace_fch4, _, _ = calc_fraction_remaining_from_acefts(ace_ch4_file, 'CH4', bc_approach='fit')
|
|
147
|
+
|
|
148
|
+
with ncdf.Dataset(ace_age_file, 'r') as nch:
|
|
149
|
+
ace_age = nch.variables['age'][:].filled(np.nan)
|
|
150
|
+
|
|
151
|
+
with ncdf.Dataset(ace_ch4_file, 'r') as nch:
|
|
152
|
+
ace_ch4_raw = nch.variables['CH4'][:].filled(np.nan)
|
|
153
|
+
ace_lat = nch.variables['latitude'][:].filled(np.nan)
|
|
154
|
+
|
|
155
|
+
logger.info('Binning F(CH4) vs. F(N2O)')
|
|
156
|
+
# We're looking for reliable stratospheric relationships. Therefore we limit to data where the concentration is
|
|
157
|
+
# positive and definitely not tropospheric (CH4 < 2e-6 i.e. < 2000 ppb), not in the polar vortex (abs(lat) < 50)
|
|
158
|
+
# and not in the mesosphere or upper stratosphere (alt < 40).
|
|
159
|
+
ace_lat = np.tile(ace_lat[:, np.newaxis], [1, ace_age.shape[1]])
|
|
160
|
+
ace_doy = np.tile(ace_doy[:, np.newaxis], [1, ace_age.shape[1]])
|
|
161
|
+
xx = ~np.isnan(ace_fch4) & ~np.isnan(ace_fn2o) & (ace_fch4 >= 0) & (ace_fn2o >= 0) & (ace_ch4_raw < 2e-6) & \
|
|
162
|
+
~mod_utils.is_vortex(ace_lat, ace_doy, ace_age) & (ace_alt[np.newaxis, :] < _tccon_top_alt)
|
|
163
|
+
|
|
164
|
+
# Define bins for F(N2O) and theta
|
|
165
|
+
fn2o_bins = _std_frac_bins
|
|
166
|
+
fn2o_bin_centers = _std_frac_bin_centers
|
|
167
|
+
|
|
168
|
+
theta_bins = _std_theta_bins
|
|
169
|
+
theta_bin_centers = _std_theta_bin_centers
|
|
170
|
+
|
|
171
|
+
# Find outliers to avoid overly noisy relationships
|
|
172
|
+
oo = _find_ch4_outliers(ace_fn2o, ace_fch4, fn2o_bins, xx)
|
|
173
|
+
|
|
174
|
+
# Do the actual binning
|
|
175
|
+
fch4_means, fch4_counts, fch4_overall, theta_overall = _bin_z_vs_xy(ace_fn2o, ace_theta, ace_fch4, xx & oo,
|
|
176
|
+
fn2o_bins, theta_bins)
|
|
177
|
+
|
|
178
|
+
logger.info('Saving F(CH4):F(N2O) file')
|
|
179
|
+
_save_fch4_lut(lut_save_file, fch4_means=fch4_means, fch4_counts=fch4_counts, fch4_overall=fch4_overall,
|
|
180
|
+
theta_overall=theta_overall, fn2o_bin_centers=fn2o_bin_centers, fn2o_bin_edges=fn2o_bins,
|
|
181
|
+
theta_bin_centers=theta_bin_centers, theta_bin_edges=theta_bins,
|
|
182
|
+
ace_ch4_file=ace_ch4_file, ace_n2o_file=ace_n2o_file, ace_age_file=ace_age_file)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def make_hf_ch4_slopes(ace_ch4_file, ace_hf_file, ace_age_file, washenfelder_supp_table_file, lut_save_file,
|
|
186
|
+
clip_values=True):
|
|
187
|
+
|
|
188
|
+
logger.info('Loading ACE data')
|
|
189
|
+
with ncdf.Dataset(ace_ch4_file, 'r') as nch_ch4, ncdf.Dataset(ace_hf_file, 'r') as nch_hf, ncdf.Dataset(ace_age_file, 'r') as nch_age:
|
|
190
|
+
ace_ch4_qual = nch_ch4.variables['quality_flag'][:].filled(9)
|
|
191
|
+
ace_hf_qual = nch_hf.variables['quality_flag'][:].filled(9)
|
|
192
|
+
|
|
193
|
+
ace_ch4 = read_ace_var(nch_ch4, 'CH4', ace_ch4_qual)
|
|
194
|
+
ace_ch4_err = read_ace_var(nch_ch4, 'CH4_error', ace_ch4_qual)
|
|
195
|
+
ace_hf = read_ace_var(nch_hf, 'HF', ace_hf_qual)
|
|
196
|
+
ace_hf_err = read_ace_var(nch_hf, 'HF_error', ace_hf_qual)
|
|
197
|
+
ace_ages = read_ace_var(nch_age, 'age', None)
|
|
198
|
+
|
|
199
|
+
ace_lat = read_ace_var(nch_ch4, 'latitude', None)
|
|
200
|
+
ace_lat = np.tile(ace_lat.reshape(-1, 1), [1, ace_ch4.shape[1]])
|
|
201
|
+
ace_alt = read_ace_var(nch_ch4, 'altitude', None)
|
|
202
|
+
ace_alt = np.tile(ace_alt, [ace_ch4.shape[0], 1])
|
|
203
|
+
ace_theta = read_ace_theta(nch_ch4, ace_ch4_qual)
|
|
204
|
+
|
|
205
|
+
ace_year = nch_ch4.variables['year'][:].filled(np.nan)
|
|
206
|
+
ace_dates = read_ace_date(nch_ch4)
|
|
207
|
+
ace_doy = np.array([mod_utils.day_of_year(d) + 1 for d in ace_dates])
|
|
208
|
+
ace_doy = ace_doy.reshape(-1, 1)
|
|
209
|
+
|
|
210
|
+
logger.info('Calculating CH4 vs. HF slopes')
|
|
211
|
+
# We do the same filtering as for the F(N2O):F(CH4) relationship, plus we require that the CH4 and HF error is < 5%.
|
|
212
|
+
# Also have to filter for excessively high HF concentrations, or it will mess up the tropics. Generally HF doesn't
|
|
213
|
+
# exceed 3 ppb and the ultra high values were all > 200 ppb, so 10 should leave all reasonable data and exclude
|
|
214
|
+
# unreasonable data (this occurred in tropics 2005)
|
|
215
|
+
#
|
|
216
|
+
# Quality filtering and fill values were handled upon reading in the data.
|
|
217
|
+
xx = ~np.isnan(ace_ch4) & ~np.isnan(ace_ch4_err) & ~np.isnan(ace_hf) & ~np.isnan(ace_hf_err) & (ace_alt < _tccon_top_alt)
|
|
218
|
+
|
|
219
|
+
if clip_values:
|
|
220
|
+
xx &= (ace_ch4 <= 2e-6) & (ace_hf < 10e-9)
|
|
221
|
+
|
|
222
|
+
# Read in the early slopes from Washenfelder et al. 2003 (doi: 10.1029/2003GL017969), table S3.
|
|
223
|
+
washenfelder_df = pd.read_csv(washenfelder_supp_table_file, header=2, sep=r'\s+')
|
|
224
|
+
washenfelder_slopes = washenfelder_df.set_index('Date').b.astype(float)
|
|
225
|
+
washenfelder_slopes.index = pd.DatetimeIndex(washenfelder_slopes.index)
|
|
226
|
+
washenfelder_slopes.name = 'slope'
|
|
227
|
+
|
|
228
|
+
# For each bin, get the slopes from the ACE-FTS data. Fit the data (including the Washenfelder slopes) to an
|
|
229
|
+
# exponential.
|
|
230
|
+
lat_bin_functions = OrderedDict([('tropics', mod_utils.is_tropics),
|
|
231
|
+
('midlat', mod_utils.is_midlat),
|
|
232
|
+
('vortex', mod_utils.is_vortex)])
|
|
233
|
+
lat_bin_slopes = pd.DataFrame(index=_get_ace_date_range(ace_dates), columns=lat_bin_functions.keys())
|
|
234
|
+
lat_bin_counts = lat_bin_slopes.copy()
|
|
235
|
+
|
|
236
|
+
fit_params = np.zeros([len(lat_bin_functions), 4], dtype=float)
|
|
237
|
+
|
|
238
|
+
for ibin, (bin_name, bin_fxn) in enumerate(lat_bin_functions.items()):
|
|
239
|
+
for year_start_date in lat_bin_slopes.index:
|
|
240
|
+
year = year_start_date.year
|
|
241
|
+
logger.debug('Generating {} slopes for {}'.format(bin_name, year))
|
|
242
|
+
lat_bin_slopes.loc[year_start_date, bin_name], lat_bin_counts.loc[year_start_date, bin_name] \
|
|
243
|
+
= _bin_and_fit_hf_vs_ch4(ace_lat=ace_lat, ace_year=ace_year, ace_dates=ace_dates, ace_doy=ace_doy,
|
|
244
|
+
ace_ages=ace_ages, ace_theta=ace_theta, ace_ch4=ace_ch4, ace_hf=ace_hf, xx=xx,
|
|
245
|
+
year=year, bin_fxn=bin_fxn)
|
|
246
|
+
|
|
247
|
+
full_series = pd.concat([washenfelder_slopes, lat_bin_slopes.loc[:, bin_name]])
|
|
248
|
+
full_series = full_series.sort_index()
|
|
249
|
+
full_series_year = full_series.index.year
|
|
250
|
+
full_series = full_series.to_numpy().astype(float)
|
|
251
|
+
|
|
252
|
+
# This was found to be a good initial guess for all latitude bins by trial and error. 3000.0 gets about the
|
|
253
|
+
# right magnitude at year 0, 0.18 get about the right decay, using the last slope as the offset generally works
|
|
254
|
+
# better than using the maximum (sometimes curve_fit has trouble moving this offset enough) and naturally we
|
|
255
|
+
# want t0 to be the first year. Trying to allow curve_fit to choose its own initial conditions ends badly.
|
|
256
|
+
p0 = [-3000.0, -0.18, full_series[-1], full_series_year[0]]
|
|
257
|
+
fit = curve_fit(mod_utils.hf_ch4_slope_fit, full_series_year, full_series, p0=p0)
|
|
258
|
+
fit_params[ibin, :] = fit[0]
|
|
259
|
+
|
|
260
|
+
# for the netCDF file, we'll include both the fits and the actual slopes for the ACE-FTS era. The default is to
|
|
261
|
+
# use the fits for all years.
|
|
262
|
+
first_year = washenfelder_slopes.index.min().year
|
|
263
|
+
last_year = lat_bin_slopes.index.max().year
|
|
264
|
+
full_dtindex = pd.date_range(start=dt.datetime(first_year, 1, 1), end=dt.datetime(last_year, 1, 1), freq='YS')
|
|
265
|
+
|
|
266
|
+
logger.info('Saving CH4 vs. HF slopes')
|
|
267
|
+
_save_hf_ch4_lut(lut_save_file, ace_ch4_file=ace_ch4_file, ace_hf_file=ace_hf_file, ace_age_file=ace_age_file,
|
|
268
|
+
lat_bin_edges=lat_bin_functions, full_date_index=full_dtindex,
|
|
269
|
+
ace_slopes=lat_bin_slopes, ace_counts=lat_bin_counts, slope_fit_params=fit_params)
|
|
270
|
+
|
|
271
|
+
return lat_bin_slopes, lat_bin_counts
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _find_ch4_outliers(ace_fn2o, ace_fch4, fn2o_bins, good_data):
|
|
275
|
+
|
|
276
|
+
# Identify outliers in each F(N2O) bin. We're not binning by theta yet; we just want to find outliers in the dataset
|
|
277
|
+
# as a whole right now. I chose m=5 since 5 sigma is the standard threshold in physics, even through we're using
|
|
278
|
+
# mean absolute deviation and not standard deviation, since the former is more robust.
|
|
279
|
+
#
|
|
280
|
+
# oo will be true for non-outlier values
|
|
281
|
+
oo = np.zeros_like(ace_fch4, dtype=np.bool_)
|
|
282
|
+
bin_inds = np.digitize(ace_fn2o, fn2o_bins) - 1
|
|
283
|
+
for b_ind in np.unique(bin_inds):
|
|
284
|
+
bb = good_data & (bin_inds == b_ind)
|
|
285
|
+
ch4_sub = ace_fch4[bb]
|
|
286
|
+
oo[bb] = ~mod_utils.isoutlier(ch4_sub, m=5)
|
|
287
|
+
|
|
288
|
+
return oo
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _bin_z_vs_xy(x, y, z, good_data, x_bins, y_bins):
|
|
292
|
+
n_x_bins = x_bins.size - 1
|
|
293
|
+
n_y_bins = y_bins.size - 1
|
|
294
|
+
x_means = np.full([n_x_bins, n_y_bins], np.nan)
|
|
295
|
+
x_counts = np.zeros_like(x_means, dtype=int)
|
|
296
|
+
|
|
297
|
+
for i, (tlow, thigh) in enumerate(zip(y_bins[:-1], y_bins[1:])):
|
|
298
|
+
aa = good_data & (y >= tlow) & (y < thigh)
|
|
299
|
+
x_sub = x[aa]
|
|
300
|
+
z_sub = z[aa]
|
|
301
|
+
bin_inds = np.digitize(x_sub, x_bins) - 1
|
|
302
|
+
|
|
303
|
+
for j in range(n_x_bins):
|
|
304
|
+
jj = bin_inds == j
|
|
305
|
+
x_means[j, i] = np.nanmean(z_sub[jj])
|
|
306
|
+
x_counts[j, i] = np.sum(jj)
|
|
307
|
+
|
|
308
|
+
z_overall = np.full([n_x_bins], np.nan)
|
|
309
|
+
y_overall = np.full([n_x_bins], np.nan)
|
|
310
|
+
bin_inds = np.digitize(x, x_bins) - 1
|
|
311
|
+
|
|
312
|
+
for j in range(n_x_bins):
|
|
313
|
+
jj = (bin_inds == j) & good_data
|
|
314
|
+
z_overall[j] = np.nanmean(z[jj])
|
|
315
|
+
y_overall[j] = np.nanmean(y[jj])
|
|
316
|
+
|
|
317
|
+
return x_means, x_counts, z_overall, y_overall
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _bin_and_fit_hf_vs_ch4(ace_lat, ace_year, ace_dates, ace_doy, ace_ages, ace_theta, ace_ch4, ace_hf, xx, year,
|
|
321
|
+
bin_fxn):
|
|
322
|
+
yy = bin_fxn(ace_lat, ace_doy, ace_ages) & (np.isclose(ace_year, year)).reshape(-1, 1)
|
|
323
|
+
|
|
324
|
+
profile_ch4_sbc = get_bc_from_ace(ace_dates=ace_dates, gas_conc=ace_ch4, theta=ace_theta, latitude=ace_lat[:, 0],
|
|
325
|
+
bc_approach='fit')
|
|
326
|
+
|
|
327
|
+
x = ace_hf[xx & yy] * 1e9
|
|
328
|
+
y = (ace_ch4 - profile_ch4_sbc.reshape(-1, 1))
|
|
329
|
+
y = y[xx & yy] * 1e9
|
|
330
|
+
|
|
331
|
+
# We use the robust fitting method with Tukey's biweighting as in Saad et al. 2014. That paper used `robustfit` from
|
|
332
|
+
# Matlab, RLM from statsmodels is the Python equivalent. We do not include an intercept because we are using the
|
|
333
|
+
# CH4 stratospheric boundary condition as the intercept, and we've already subtracted that.
|
|
334
|
+
slope = RLM(y, x, M=TukeyBiweight()).fit().params.item()
|
|
335
|
+
|
|
336
|
+
return slope, np.sum(xx & yy)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _get_year_avg_ch4_sbc(year, ch4):
|
|
340
|
+
tt = (ch4.conc_seasonal.index >= dt.datetime(year, 1, 1)) & (ch4.conc_seasonal.index < dt.datetime(year + 1, 1, 1))
|
|
341
|
+
return ch4.conc_seasonal.dmf_mean[tt].mean()
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _save_fn2o_lut(nc_filename, fn2o_means, fn2o_counts, fn2o_overall, theta_overall,
|
|
345
|
+
age_bin_centers, age_bin_edges, theta_bin_centers, theta_bin_edges,
|
|
346
|
+
ace_n2o_file, ace_age_file):
|
|
347
|
+
|
|
348
|
+
with ncdf.Dataset(nc_filename, 'w') as nch:
|
|
349
|
+
age_dim = ioutils.make_ncdim_helper(nch, 'age', age_bin_centers,
|
|
350
|
+
description='Age of stratospheric air from the CLaMS model',
|
|
351
|
+
units='years')
|
|
352
|
+
ioutils.make_ncdim_helper(nch, 'age_bins', age_bin_edges,
|
|
353
|
+
description='Edges of age bins used when binning the F(N2O) data',
|
|
354
|
+
units='years')
|
|
355
|
+
theta_dim = ioutils.make_ncdim_helper(nch, 'theta', theta_bin_centers,
|
|
356
|
+
description='Potential temperature',
|
|
357
|
+
units='K')
|
|
358
|
+
ioutils.make_ncdim_helper(nch, 'theta_bins', theta_bin_edges,
|
|
359
|
+
description='Edges of potential temperature bins used when binning the F(N2O) data',
|
|
360
|
+
units='K')
|
|
361
|
+
|
|
362
|
+
ioutils.make_ncvar_helper(nch, 'fn2o', fn2o_means, (age_dim, theta_dim),
|
|
363
|
+
description='Mean value of F(N2O) (fraction of N2O remaining relative to the '
|
|
364
|
+
'stratospheric boundary condition) in the age/theta bin',
|
|
365
|
+
units='unitless')
|
|
366
|
+
ioutils.make_ncvar_helper(nch, 'fn2o_counts', fn2o_counts, (age_dim, theta_dim),
|
|
367
|
+
description='Number of F(N2O) observations in each age/theta bin',
|
|
368
|
+
units='number')
|
|
369
|
+
ioutils.make_ncvar_helper(nch, 'fn2o_overall', fn2o_overall, (age_dim,),
|
|
370
|
+
description='Mean value of F(N2O) for a given age bin, not separated by theta',
|
|
371
|
+
units='unitless')
|
|
372
|
+
ioutils.make_ncvar_helper(nch, 'theta_overall', theta_overall, (age_dim,),
|
|
373
|
+
description='Mean value of potential temperature in a given age bin',
|
|
374
|
+
units='K')
|
|
375
|
+
ioutils.add_dependent_file_hash(nch, 'ace_n2o_file_sha1', ace_n2o_file)
|
|
376
|
+
ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
|
|
377
|
+
ioutils.add_creation_info(nch, 'make_fn2o_lookup_table')
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _save_fch4_lut(nc_filename, fch4_means, fch4_counts, fch4_overall, theta_overall,
|
|
381
|
+
fn2o_bin_centers, fn2o_bin_edges, theta_bin_centers, theta_bin_edges,
|
|
382
|
+
ace_ch4_file, ace_n2o_file, ace_age_file):
|
|
383
|
+
with ncdf.Dataset(nc_filename, 'w') as nch:
|
|
384
|
+
fn2o_dim = ioutils.make_ncdim_helper(nch, 'fn2o', fn2o_bin_centers,
|
|
385
|
+
description='Fraction of N2O remaining relative to the stratospheric boundary condition.',
|
|
386
|
+
units='unitless')
|
|
387
|
+
ioutils.make_ncdim_helper(nch, 'fn2o_bins', fn2o_bin_edges,
|
|
388
|
+
description='Edges of F(N2O) bins used when binning the F(CH4) data',
|
|
389
|
+
units='unitless')
|
|
390
|
+
theta_dim = ioutils.make_ncdim_helper(nch, 'theta', theta_bin_centers,
|
|
391
|
+
description='Potential temperature',
|
|
392
|
+
units='K')
|
|
393
|
+
ioutils.make_ncdim_helper(nch, 'theta_bins', theta_bin_edges,
|
|
394
|
+
description='Edges of potential temperature bins used when binning the F(CH4) data',
|
|
395
|
+
units='K')
|
|
396
|
+
|
|
397
|
+
ioutils.make_ncvar_helper(nch, 'fch4', fch4_means, (fn2o_dim, theta_dim),
|
|
398
|
+
description='Mean value of F(CH4) (fraction of CH4 remaining relative to the '
|
|
399
|
+
'stratospheric boundary condition) in the F(N2O)/theta bin',
|
|
400
|
+
units='unitless')
|
|
401
|
+
ioutils.make_ncvar_helper(nch, 'fch4_counts', fch4_counts, (fn2o_dim, theta_dim),
|
|
402
|
+
description='Number of CH4 observations in each F(N2O)/theta bin',
|
|
403
|
+
units='number')
|
|
404
|
+
ioutils.make_ncvar_helper(nch, 'fch4_overall', fch4_overall, (fn2o_dim,),
|
|
405
|
+
description='Mean value of F(CH4) for a given F(N2O) bin, not separated by theta',
|
|
406
|
+
units='unitless')
|
|
407
|
+
ioutils.make_ncvar_helper(nch, 'theta_overall', theta_overall, (fn2o_dim,),
|
|
408
|
+
description='Mean value of potential temperature in a given F(N2O) bin',
|
|
409
|
+
units='K')
|
|
410
|
+
ioutils.add_creation_info(nch, 'make_fch4_fn2o_lookup_table')
|
|
411
|
+
ioutils.add_dependent_file_hash(nch, 'ace_ch4_file_sha1', ace_ch4_file)
|
|
412
|
+
ioutils.add_dependent_file_hash(nch, 'ace_n2o_file_sha1', ace_n2o_file)
|
|
413
|
+
ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _save_hf_ch4_lut(nc_filename, ace_ch4_file, ace_hf_file, ace_age_file, lat_bin_edges,
|
|
417
|
+
full_date_index, ace_slopes, ace_counts, slope_fit_params):
|
|
418
|
+
with ncdf.Dataset(nc_filename, 'w') as nch:
|
|
419
|
+
ioutils.add_creation_info(nch, creation_note='make_hf_ch4_slopes')
|
|
420
|
+
ioutils.add_dependent_file_hash(nch, 'ace_ch4_file_sha1', ace_ch4_file)
|
|
421
|
+
ioutils.add_dependent_file_hash(nch, 'ace_hf_file_sha1', ace_hf_file)
|
|
422
|
+
ioutils.add_dependent_file_hash(nch, 'ace_age_file_sha1', ace_age_file)
|
|
423
|
+
|
|
424
|
+
nch.ace_ch4_file = ace_ch4_file
|
|
425
|
+
nch.ace_hf_file = ace_hf_file
|
|
426
|
+
|
|
427
|
+
ch4_hf_slopes = pd.DataFrame(index=full_date_index, columns=ace_slopes.columns)
|
|
428
|
+
ch4_hf_count = pd.DataFrame(index=full_date_index, columns=ace_counts.columns).fillna(0)
|
|
429
|
+
ch4_hf_source = pd.DataFrame(index=full_date_index, columns=ace_slopes.columns).fillna(1)
|
|
430
|
+
# transpose so that the bins dim is second like the above dataframes
|
|
431
|
+
bin_names_char_array = ncdf.stringtochar(np.array(list(lat_bin_edges.keys()))).T
|
|
432
|
+
slope_fit_params = slope_fit_params.T
|
|
433
|
+
|
|
434
|
+
# For each latitude bin, make a pandas series for the full date index from the fit first, then replace available
|
|
435
|
+
# years with the actual fit
|
|
436
|
+
for ibin, bin_name in enumerate(lat_bin_edges.keys()):
|
|
437
|
+
ch4_hf_slopes.loc[:, bin_name] = mod_utils.hf_ch4_slope_fit(ch4_hf_slopes.index.year, *slope_fit_params[:, ibin])
|
|
438
|
+
ch4_hf_slopes.loc[ace_slopes.index, bin_name] = ace_slopes.loc[:, bin_name]
|
|
439
|
+
ch4_hf_count.loc[ace_counts.index, bin_name] = ace_counts.loc[:, bin_name]
|
|
440
|
+
ch4_hf_source.loc[ace_slopes.index, bin_name] = 0
|
|
441
|
+
|
|
442
|
+
# Setup dimensions.
|
|
443
|
+
nbins = len(lat_bin_edges)
|
|
444
|
+
nchars = bin_names_char_array.shape[0]
|
|
445
|
+
nparams = slope_fit_params.shape[0]
|
|
446
|
+
year_dim = ioutils.make_ncdim_helper(nch, 'year', full_date_index.year.to_numpy())
|
|
447
|
+
bins_dim = nch.createDimension('latitude_bins', size=nbins)
|
|
448
|
+
char_dim = nch.createDimension('bin_name_length', size=nchars)
|
|
449
|
+
fit_params_dim = nch.createDimension('exp_fit_params_length', size=nparams)
|
|
450
|
+
|
|
451
|
+
# Save the variables
|
|
452
|
+
ioutils.make_ncvar_helper(nch, 'ch4_hf_slopes', ch4_hf_slopes.to_numpy().astype(float), [year_dim, bins_dim],
|
|
453
|
+
units='ppb CH4/ppb HF',
|
|
454
|
+
description='Slope of stratospheric CH4 concentrations vs HF concentrations from '
|
|
455
|
+
'ACE-FTS data for specific years and latitude bins')
|
|
456
|
+
ioutils.make_ncvar_helper(nch, 'ace_counts', ch4_hf_count.to_numpy().astype(float), [year_dim, bins_dim],
|
|
457
|
+
units='#',
|
|
458
|
+
description='Number of ACE-FTS observations used to calculate this slope')
|
|
459
|
+
ioutils.make_ncvar_helper(nch, 'ch4_hf_slopes_source', ch4_hf_source.to_numpy().astype(int), [year_dim, bins_dim],
|
|
460
|
+
units='N/A',
|
|
461
|
+
description='Flag indicating how the slope was computed. 0 means taken directly from '
|
|
462
|
+
'fits of ACE-FTS CH4 vs. HF; 1 means sampled from the exponential fit '
|
|
463
|
+
'of ACE-FTS + Washenfelder 03 slopes')
|
|
464
|
+
ioutils.make_ncvar_helper(nch, 'slope_fit_params', slope_fit_params, [fit_params_dim, bins_dim],
|
|
465
|
+
description='Fitting parameters a, b, c, and t0 for a function of form '
|
|
466
|
+
'"a * exp(b*(t - t0)) + c" used to fit the temporal trend of CH4 vs. HF '
|
|
467
|
+
'slopes, where t is the year (e.g. 2004)')
|
|
468
|
+
ioutils.make_ncvar_helper(nch, 'bin_names', bin_names_char_array, [char_dim, bins_dim],
|
|
469
|
+
description='The names used for the latitude bins')
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def plot_hf_ch4_slopes(lut_file, years=np.arange(1970, 2019), ax=None):
|
|
473
|
+
if ax is None:
|
|
474
|
+
fig, ax = plt.subplots()
|
|
475
|
+
else:
|
|
476
|
+
fig = ax.figure
|
|
477
|
+
colors = ['r', 'g', 'b']
|
|
478
|
+
linestyles = ['-', '--', ':']
|
|
479
|
+
with xr.open_dataset(lut_file) as ds:
|
|
480
|
+
lat_bins = ds['latitude_bins'][:]
|
|
481
|
+
params = ds['slope_fit_params'][:]
|
|
482
|
+
bin_names = ds['bin_names'][:]
|
|
483
|
+
for i, col, style in zip(lat_bins, colors, linestyles):
|
|
484
|
+
label = bin_names.sel(latitude_bins=i).data
|
|
485
|
+
try:
|
|
486
|
+
label = ''.join(label)
|
|
487
|
+
except TypeError:
|
|
488
|
+
label = b''.join(label).decode('utf8')
|
|
489
|
+
|
|
490
|
+
slopes = mod_utils.hf_ch4_slope_fit(years, *params.sel(latitude_bins=i).data)
|
|
491
|
+
ax.plot(years, slopes, color=col, linestyle=style, label=label)
|
|
492
|
+
|
|
493
|
+
ax.grid()
|
|
494
|
+
ax.set_ylabel('CH4:HF slope from fit')
|
|
495
|
+
ax.legend()
|
|
496
|
+
|
|
497
|
+
return fig, ax
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def get_bc_from_ace(ace_dates, gas_conc, theta, latitude, bc_approach='fit'):
|
|
501
|
+
"""
|
|
502
|
+
Calcaulate the stratospheric boundary condition for each ACE profile from ACE data itself
|
|
503
|
+
|
|
504
|
+
:param ace_dates: a 1D array of datetimes with one entry per ACE profile
|
|
505
|
+
:type ace_dates: array-like
|
|
506
|
+
|
|
507
|
+
:param gas_conc: the array of ACE concentrations for the gas of interest. May be any units, the boundary condition
|
|
508
|
+
values will be returned in the same units.
|
|
509
|
+
:type gas_conc: array-like
|
|
510
|
+
|
|
511
|
+
:param theta: the array of ACE potential temperatures.
|
|
512
|
+
:type theta: array-like
|
|
513
|
+
|
|
514
|
+
:param latitude: the array of ACE profile latitudes (1D)
|
|
515
|
+
:type latitude: array-like
|
|
516
|
+
|
|
517
|
+
:param bc_approach: how to calculate the boundary condition. "fit" (default) fits all ACE data in the tropics
|
|
518
|
+
between 360 and 390 K to generate a quadratic fit over time. "per-profile" calculates the boundary condition for
|
|
519
|
+
each profile using its own data between 360 and 390 K.
|
|
520
|
+
:type bc_approach: str
|
|
521
|
+
|
|
522
|
+
:return: the array of boundary condition concentrations, one per profile.
|
|
523
|
+
"""
|
|
524
|
+
bottom_pt = 360
|
|
525
|
+
top_pt = 390
|
|
526
|
+
|
|
527
|
+
def _get_bc_per_profile(dates, concentration, pt, lat):
|
|
528
|
+
bc_conc = np.full([concentration.shape[0]], np.nan)
|
|
529
|
+
for i, (conc_row, pt_row) in enumerate(zip(concentration, pt)):
|
|
530
|
+
zz = (pt_row > bottom_pt) & (pt_row < top_pt)
|
|
531
|
+
|
|
532
|
+
if zz.sum() > 0:
|
|
533
|
+
bc_conc[i] = np.nanmean(conc_row[zz])
|
|
534
|
+
|
|
535
|
+
return bc_conc
|
|
536
|
+
|
|
537
|
+
def _get_bc_by_fit(dates, concentration, pt, lat):
|
|
538
|
+
datenums = np.full_like(concentration, np.nan)
|
|
539
|
+
datenums[:] = np.array([np.datetime64(d) for d in dates]).reshape(-1, 1)
|
|
540
|
+
in_pt = (pt > 360) & (pt < 390)
|
|
541
|
+
in_tropics = mod_utils.is_tropics(lat, None, None)
|
|
542
|
+
in_both = in_pt & in_tropics.reshape(-1, 1)
|
|
543
|
+
bc_concs = concentration[in_both]
|
|
544
|
+
bc_datenums = datenums[in_both]
|
|
545
|
+
not_outliers = ~mod_utils.isoutlier(bc_concs, m=5)
|
|
546
|
+
not_nans = ~np.isnan(bc_datenums) & ~np.isnan(bc_concs)
|
|
547
|
+
xx = not_outliers & not_nans
|
|
548
|
+
fit = np.polynomial.polynomial.Polynomial.fit(bc_datenums[xx], bc_concs[xx], deg=2)
|
|
549
|
+
return fit(datenums[:, 0])
|
|
550
|
+
|
|
551
|
+
if bc_approach == 'per-profile':
|
|
552
|
+
bc_fxn = _get_bc_per_profile
|
|
553
|
+
elif bc_approach == 'fit':
|
|
554
|
+
bc_fxn = _get_bc_by_fit
|
|
555
|
+
else:
|
|
556
|
+
raise ValueError('bc_approach must be "per-profile" or "fit"')
|
|
557
|
+
|
|
558
|
+
return bc_fxn(ace_dates, gas_conc, theta, latitude)
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def calc_fraction_remaining_from_acefts(nc_file, gas_name, bc_approach='per-profile'):
|
|
562
|
+
"""
|
|
563
|
+
Calculate the fraction remaining of a gas in the stratosphere from an ACE-FTS netCDF file
|
|
564
|
+
|
|
565
|
+
:param nc_file: the path to the ACE-FTS netCDF file
|
|
566
|
+
:type nc_file: str
|
|
567
|
+
|
|
568
|
+
:param gas_name: the variable name in the netCDF file that holds the gas concentrations
|
|
569
|
+
:type gas_name: str
|
|
570
|
+
|
|
571
|
+
:param gas_record: a subclass instance of :class:`TropicsTraceGasRecord` that provides the stratospheric boundary
|
|
572
|
+
condition for the given gas.
|
|
573
|
+
:type gas_record: :class:`tccon_priors.TropicsTraceGasRecord`
|
|
574
|
+
|
|
575
|
+
:param tropopause_approach: how to find the tropopause. Options are:
|
|
576
|
+
|
|
577
|
+
* 'wmo' - uses the WMO definition, looking for lapse rate < 2 K/km
|
|
578
|
+
* 'theta' - finds the altitude at which potential temperature is 380 K
|
|
579
|
+
|
|
580
|
+
:type tropopause_approach: str
|
|
581
|
+
|
|
582
|
+
:return: the vector of altitudes that the ACE-FTS profiles are defined on and the array of fraction of gas
|
|
583
|
+
remaining. The latter will be set to NaN in the troposphere.
|
|
584
|
+
:rtype: :class:`numpy.ndarray` x2
|
|
585
|
+
"""
|
|
586
|
+
|
|
587
|
+
with ncdf.Dataset(nc_file, 'r') as nch:
|
|
588
|
+
alt = read_ace_var(nch, 'altitude', qflags=None)
|
|
589
|
+
latitude = read_ace_var(nch, 'latitude', qflags=None)
|
|
590
|
+
ace_dates = read_ace_date(nch)
|
|
591
|
+
qflags = read_ace_var(nch, 'quality_flag', qflags=None)
|
|
592
|
+
|
|
593
|
+
gas_conc = read_ace_var(nch, gas_name, qflags=qflags)
|
|
594
|
+
# ACE data appears to use -999 as fill value in the concentrations
|
|
595
|
+
gas_conc[gas_conc < -900.0] = np.nan
|
|
596
|
+
|
|
597
|
+
theta = read_ace_theta(nch, qflags)
|
|
598
|
+
|
|
599
|
+
bc_concentrations = get_bc_from_ace(ace_dates, gas_conc, theta, latitude, bc_approach=bc_approach)
|
|
600
|
+
fgas = gas_conc / bc_concentrations.reshape(-1, 1)
|
|
601
|
+
fgas[theta < 380] = np.nan
|
|
602
|
+
|
|
603
|
+
return alt, fgas, theta, {'dates': ace_dates, 'bc_concs': bc_concentrations, 'gas_concs': gas_conc}
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _get_ace_date_range(ace_dates, freq='YS'):
|
|
607
|
+
min_date = np.min(ace_dates)
|
|
608
|
+
if min_date < dt.datetime(min_date.year, 4, 1):
|
|
609
|
+
start_year = min_date.year
|
|
610
|
+
else:
|
|
611
|
+
start_year = min_date.year + 1
|
|
612
|
+
|
|
613
|
+
max_date = np.max(ace_dates)
|
|
614
|
+
if max_date >= dt.datetime(max_date.year, 10, 1):
|
|
615
|
+
end_year = max_date.year
|
|
616
|
+
else:
|
|
617
|
+
end_year = max_date.year - 1
|
|
618
|
+
|
|
619
|
+
return pd.date_range(start=dt.datetime(start_year, 1, 1), end=dt.datetime(end_year, 1, 1), freq=freq)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def add_clams_age_to_file(ace_nc_file, save_nc_file=None):
|
|
623
|
+
"""
|
|
624
|
+
Calculate CLaMS age for profiles in an ACE-FTS files. Save either to the given ACE file or a new file.
|
|
625
|
+
|
|
626
|
+
Note: this function does not calculate equivalent latitude for the ACE profiles before computing the age, it simply
|
|
627
|
+
uses the geographic latitude of the ACE profiles.
|
|
628
|
+
|
|
629
|
+
:param ace_nc_file: the ACE-FTS netCDF file to compute ages for.
|
|
630
|
+
:type ace_nc_file: str
|
|
631
|
+
|
|
632
|
+
:param save_nc_file: optional, if given a path to a file, the ages will be saved to this file (which is overwritten)
|
|
633
|
+
along with the altitude dimension. If not given (or ``None``), the ages are added to the file specified by
|
|
634
|
+
``ace_nc_file``.
|
|
635
|
+
:type save_nc_file: str
|
|
636
|
+
|
|
637
|
+
:return: None
|
|
638
|
+
"""
|
|
639
|
+
with ncdf.Dataset(ace_nc_file, 'r') as nch:
|
|
640
|
+
ace_lat = read_ace_var(nch, 'latitude', None)
|
|
641
|
+
ace_alt = read_ace_var(nch, 'altitude', None)
|
|
642
|
+
ace_dates = read_ace_date(nch)
|
|
643
|
+
ace_doy = np.array([mod_utils.day_of_year(d) + 1 for d in ace_dates])
|
|
644
|
+
ace_theta = read_ace_theta(nch, None)
|
|
645
|
+
|
|
646
|
+
ace_ages = np.full_like(ace_theta, np.nan)
|
|
647
|
+
pbar = mod_utils.ProgressBar(ace_theta.shape[0], prefix='Calculating ages for profile', style='counter')
|
|
648
|
+
for iprof in range(ace_theta.shape[0]):
|
|
649
|
+
pbar.print_bar(iprof)
|
|
650
|
+
prof_theta = ace_theta[iprof]
|
|
651
|
+
prof_lat = np.full_like(prof_theta, ace_lat[iprof])
|
|
652
|
+
prof_doy = ace_doy[iprof]
|
|
653
|
+
ace_ages[iprof, :] = tccon_priors.get_clams_age(prof_theta, prof_lat, prof_doy)
|
|
654
|
+
|
|
655
|
+
pbar.finish()
|
|
656
|
+
|
|
657
|
+
if save_nc_file is None:
|
|
658
|
+
logger.info('Added CLaMS age to input file: {}'.format(ace_nc_file))
|
|
659
|
+
with ncdf.Dataset(ace_nc_file, 'a') as nch:
|
|
660
|
+
ioutils.make_ncvar_helper(nch, 'age', ace_ages, ['index', 'altitude'], units='years',
|
|
661
|
+
description='Age from the CLaMS model along the ACE profiles')
|
|
662
|
+
else:
|
|
663
|
+
logger.info('Saving CLaMS age to new file: {}'.format(save_nc_file))
|
|
664
|
+
with ncdf.Dataset(save_nc_file, 'w') as nch:
|
|
665
|
+
index_dim = nch.createDimension('index', ace_ages.shape[0])
|
|
666
|
+
alt_dim = ioutils.make_ncdim_helper(nch, 'altitude', ace_alt, long_name='altitude',
|
|
667
|
+
description='Fixed altitude grid', units='km')
|
|
668
|
+
ioutils.make_ncvar_helper(nch, 'age', ace_ages, [index_dim, alt_dim], units='years', long_name='clams_age',
|
|
669
|
+
description='Age from the CLaMS model along the ACE profiles')
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def generate_ace_age_file(ace_in_file, age_file_out, geos_path, use_geos_theta_for_age=False, geos_product='fpit', allow_missing_geos=False, nprocs=0):
|
|
673
|
+
"""Create a file that mimics the structure of an ACE-FTS v3 file but stores age of air and equivalent latitude
|
|
674
|
+
|
|
675
|
+
Parameters
|
|
676
|
+
----------
|
|
677
|
+
ace_in_file: str
|
|
678
|
+
Path to any of the original ACE files. It doesn't need to be any one in particular; all should have the lat,
|
|
679
|
+
lon, pressure, temperature, quality flag, and date information needed. However, if this is for a particular
|
|
680
|
+
gas, you may as well pass that gas's file, in case the quality flags differ.
|
|
681
|
+
|
|
682
|
+
age_file_out: str
|
|
683
|
+
Path to write the resulting file to (the file itself, not a directory).
|
|
684
|
+
|
|
685
|
+
geos_path: str
|
|
686
|
+
Path to the directory containing the GEOS met data. This directory must contain an "Nv" subdirectory with the
|
|
687
|
+
3D met files.
|
|
688
|
+
|
|
689
|
+
use_geos_theta_for_age: bool
|
|
690
|
+
Set to ``True`` to use potential temperature calculated from the GEOS files, rather than ACE, when determining
|
|
691
|
+
the age of air.
|
|
692
|
+
|
|
693
|
+
geos_product: str
|
|
694
|
+
A string, usually "fpit", "fp", or "it" that describes which GEOS product to read. This is necessary to determine
|
|
695
|
+
the file names.
|
|
696
|
+
|
|
697
|
+
allow_missing_geos: bool
|
|
698
|
+
Set to ``True`` to skip profiles for which the necessary GEOS files are not available. Otherwise, any missing GEOS
|
|
699
|
+
file is an error.
|
|
700
|
+
|
|
701
|
+
nprocs: int
|
|
702
|
+
How many processors to use. <= 1 will run in serial, >= 2 will use multiprocessing to run in parallel.
|
|
703
|
+
"""
|
|
704
|
+
# This will need to group ACE profiles by which GEOS files they fall between, read the PV from those files,
|
|
705
|
+
# interpolate to the ACE profile lat/lon/time, generate the EL interpolators, calculate the EL profiles, then
|
|
706
|
+
# finally lookup age from CLAMS.
|
|
707
|
+
logger.info('Generating ACE age file with NATIVE-level GEOS files')
|
|
708
|
+
ace_data = dict()
|
|
709
|
+
with ncdf.Dataset(ace_in_file) as nh:
|
|
710
|
+
ace_data['dates'] = read_ace_date(nh)
|
|
711
|
+
ace_qflags = read_ace_var(nh, 'quality_flag', None)
|
|
712
|
+
# This wraps lons outside [-180,180] and lats outside [-90,90] into
|
|
713
|
+
# those ranges
|
|
714
|
+
ace_data['lon'], ace_data['lat'] = read_ace_latlon(nh, clip=True)
|
|
715
|
+
ace_data['theta'] = read_ace_theta(nh, qflags=ace_qflags)
|
|
716
|
+
|
|
717
|
+
date_groups = _bin_ace_to_geos_times(ace_data['dates'])
|
|
718
|
+
|
|
719
|
+
# Now we need to loop over the date groups and calculate the eq. lat. and age for each group (since each group
|
|
720
|
+
# is the ace profiles in a 3 hour window bracketed by 2 GEOS files). There's usually only a few profiles per group,
|
|
721
|
+
# so it makes more sense to parallelize over groups than profiles within groups.
|
|
722
|
+
#
|
|
723
|
+
# Even though it may be memory intensive, we're going to build the inputs for each call to _calc_el_age_for_ace
|
|
724
|
+
# first then iterate so that we can use the same inputs for serial or parallel mode. We'll build as iterators to
|
|
725
|
+
# limit the memory use as much as possible.
|
|
726
|
+
group_inputs = dict()
|
|
727
|
+
group_inputs['ace_dates'] = (ace_data['dates'][xx] for xx in date_groups.values())
|
|
728
|
+
group_inputs['ace_lons'] = (ace_data['lon'][xx] for xx in date_groups.values())
|
|
729
|
+
group_inputs['ace_lats'] = (ace_data['lat'][xx] for xx in date_groups.values())
|
|
730
|
+
group_inputs['ace_theta'] = (ace_data['theta'][xx, :] for xx in date_groups.values())
|
|
731
|
+
group_inputs['geos_dates'] = ((d, d+dt.timedelta(hours=3)) for d in date_groups.keys())
|
|
732
|
+
group_inputs['geos_path'] = repeat(geos_path)
|
|
733
|
+
group_inputs['use_geos_theta'] = repeat(use_geos_theta_for_age)
|
|
734
|
+
group_inputs['geos_product'] = repeat(geos_product)
|
|
735
|
+
group_inputs['allow_missing'] = repeat(allow_missing_geos)
|
|
736
|
+
|
|
737
|
+
input_order = ('ace_dates', 'ace_lons', 'ace_lats', 'ace_theta', 'geos_dates', 'geos_path', 'use_geos_theta', 'geos_product', 'allow_missing')
|
|
738
|
+
inputs_interator = zip(*[group_inputs[k] for k in input_order])
|
|
739
|
+
|
|
740
|
+
if nprocs <= 1:
|
|
741
|
+
results = []
|
|
742
|
+
for inputs in inputs_interator:
|
|
743
|
+
results.append(_calc_el_age_for_ace(*inputs))
|
|
744
|
+
else:
|
|
745
|
+
with Pool(processes=nprocs) as pool:
|
|
746
|
+
results = pool.starmap(_calc_el_age_for_ace, inputs_interator)
|
|
747
|
+
|
|
748
|
+
# _calc_el_age_for_age returns a dictionary of nprofs-by-nlevels arrays, where nprofs is the number of profiles in
|
|
749
|
+
# the group it was given. results is then a list of these dictionaries, so we need to go through those dictionaries
|
|
750
|
+
# and put their subset of profiles in the right rows in the bigger array
|
|
751
|
+
output_keys = ('EL', 'age')
|
|
752
|
+
ace_outputs = {k: np.full_like(ace_data['theta'], np.nan) for k in output_keys}
|
|
753
|
+
for i, inds in enumerate(date_groups.values()):
|
|
754
|
+
if results[i] is None and allow_missing_geos:
|
|
755
|
+
continue
|
|
756
|
+
elif results[i] is None and not allow_missing_geos:
|
|
757
|
+
raise TypeError('Got a None value when missing GEOS files were forbidden')
|
|
758
|
+
|
|
759
|
+
for k in output_keys:
|
|
760
|
+
ace_outputs[k][inds] = results[i][k]
|
|
761
|
+
|
|
762
|
+
save_ace_age_file(age_file_out, ('longitude', 'latitude', 'orbit', 'year', 'month', 'day', 'hour', 'quality_flag', 'temperature', 'pressure'),
|
|
763
|
+
ace_in_file, ace_outputs['EL'], ace_outputs['age'])
|
|
764
|
+
logger.info('Done generating ACE age file with NATIVE-level GEOS files')
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def _calc_el_age_for_ace(ace_dates, ace_lon, ace_lat, ace_theta, geos_dates, geos_path, use_geos_theta_for_age=False, geos_product='fpit', allow_missing=False):
|
|
768
|
+
# Let's get to the point where we load the GEOS data
|
|
769
|
+
date_str = ', '.join(d.strftime('%Y-%m-%d %H:%M') for d in ace_dates)
|
|
770
|
+
logger.info('Calculating eqlat and age for profiles on {}'.format(date_str))
|
|
771
|
+
# Read in the GEOS data, calculating quantities as necessary #
|
|
772
|
+
geos_vars = {'EPV': 1e6, 'T': 1.0, 'DELP': 1.0}
|
|
773
|
+
|
|
774
|
+
try:
|
|
775
|
+
geos_files = [os.path.join(geos_path, 'Nv', mod_utils._format_geosfp_name(geos_product, 'met', 'eta', d)) for d in geos_dates]
|
|
776
|
+
missing_files = ', '.join(f for f in geos_files if not os.path.exists(f))
|
|
777
|
+
if missing_files:
|
|
778
|
+
if allow_missing:
|
|
779
|
+
date_range = f'{min(ace_dates)} to {max(ace_dates)}'
|
|
780
|
+
logger.warn(f'Missing at least one GEOS file for {date_range}: {missing_files}')
|
|
781
|
+
return None
|
|
782
|
+
else:
|
|
783
|
+
raise IOError(f'Missing GEOS files: {missing_files}')
|
|
784
|
+
geos_data_on_std_times = []
|
|
785
|
+
geos_pres = []
|
|
786
|
+
for i, f in enumerate(geos_files):
|
|
787
|
+
this_data = _interp_geos_vars_to_ace_lat_lon(f, geos_vars, ace_lon, ace_lat)
|
|
788
|
+
geos_pres.append(np.fliplr(mod_utils.convert_geos_eta_coord(this_data.pop('DELP'))))
|
|
789
|
+
|
|
790
|
+
# Handle flipping vertically after computing pressure in case the convert function expects the
|
|
791
|
+
# delp vector space-to-surface. Flipping lr instead of ud because the flip has to go along the
|
|
792
|
+
# second dimension
|
|
793
|
+
for k, v in this_data.items():
|
|
794
|
+
this_data[k] = np.fliplr(v)
|
|
795
|
+
|
|
796
|
+
geos_data_on_std_times.append(this_data)
|
|
797
|
+
|
|
798
|
+
# Generate the eq. lat. intepolators and find the eq. lat. profiles on the GEOS levels #
|
|
799
|
+
eqlat_interpolators = mm.equivalent_latitude_functions_from_native_geos_files(geos_files, geos_dates, muted=True)
|
|
800
|
+
for i, geos_data in enumerate(geos_data_on_std_times):
|
|
801
|
+
gdate = geos_dates[i]
|
|
802
|
+
geos_data['PT'] = mod_utils.calculate_potential_temperature(geos_pres[i], geos_data['T'])
|
|
803
|
+
# Calculate the equivalent latitude profiles.
|
|
804
|
+
eqlat = np.full_like(geos_data['T'], np.nan)
|
|
805
|
+
for j in range(ace_lon.size):
|
|
806
|
+
eqlat[j, :] = mod_utils.get_eqlat_profile(eqlat_interpolators[gdate], geos_data['EPV'][j, :], geos_data['PT'][j, :])
|
|
807
|
+
geos_data['EL'] = eqlat
|
|
808
|
+
|
|
809
|
+
# Put the profiles onto the ACE times and levels #
|
|
810
|
+
geos_data_at_ace_times = _interp_geos_vars_to_ace_times(ace_dates, geos_dates, geos_data_on_std_times)
|
|
811
|
+
geos_data_on_ace_levels = dict()
|
|
812
|
+
for varname, geos_data in geos_data_at_ace_times.items():
|
|
813
|
+
ace_profiles = np.full_like(ace_theta, np.nan)
|
|
814
|
+
for i, _ in enumerate(geos_data):
|
|
815
|
+
# Limit to GEOS levels where potential temperature is monotonically increasing.
|
|
816
|
+
decr_theta = np.diff(geos_data_at_ace_times['PT'][i,:]) <= 0
|
|
817
|
+
if np.any(decr_theta):
|
|
818
|
+
pt_incr = np.flatnonzero(decr_theta)[-1] + 1
|
|
819
|
+
logger.debug('Limiting GEOS data to level {} and up to ensure monotonically increasing potential temperature'.format(pt_incr+1))
|
|
820
|
+
else:
|
|
821
|
+
pt_incr = 0
|
|
822
|
+
logger.debug('GEOS potential temperature monotonically increasing for all levels')
|
|
823
|
+
# Check that the GEOS data still covers the range of ACE potential temperatures. It should be okay if it doesn't
|
|
824
|
+
# (ACE should just get NaNs there) this is just to see how often it happens
|
|
825
|
+
min_geos_pt = np.nanmin(geos_data_at_ace_times['PT'][i, pt_incr:])
|
|
826
|
+
min_ace_pt = np.nanmin(ace_theta[i, :])
|
|
827
|
+
if min_geos_pt > min_ace_pt:
|
|
828
|
+
logger.info('Note that GEOS PT starts increasing monotonically above the bottom ACE level (profile time: {date}, ACE bottom: {ace:.2f}, GEOS bottom: {geos:.2f})'
|
|
829
|
+
.format(date=ace_dates[i], ace=min_ace_pt, geos=min_geos_pt))
|
|
830
|
+
|
|
831
|
+
# This is one case where I think we do not want to extrapolate because the ACE data spans much greater
|
|
832
|
+
# vertical extent than the GEOS data, so it can't be reliable
|
|
833
|
+
ace_profiles[i, :] = np.interp(ace_theta[i, :], geos_data_at_ace_times['PT'][i, pt_incr:], geos_data[i, pt_incr:],
|
|
834
|
+
left=np.nan, right=np.nan)
|
|
835
|
+
geos_data_on_ace_levels[varname] = ace_profiles
|
|
836
|
+
|
|
837
|
+
# Use the equivalent latitude and potential temperature profiles to look up CLAMS ages #
|
|
838
|
+
ace_age = np.full_like(ace_theta, np.nan)
|
|
839
|
+
for i in range(ace_lon.size):
|
|
840
|
+
if use_geos_theta_for_age:
|
|
841
|
+
theta_prof = geos_data_on_ace_levels['PT'][i, :]
|
|
842
|
+
else:
|
|
843
|
+
theta_prof = ace_theta[i, :]
|
|
844
|
+
|
|
845
|
+
eqlat_prof = geos_data_on_ace_levels['EL'][i, :]
|
|
846
|
+
ace_doy = mod_utils.clams_day_of_year(ace_dates[i])
|
|
847
|
+
|
|
848
|
+
ace_age[i, :] = tccon_priors.get_clams_age(theta_prof, eqlat_prof, ace_doy)
|
|
849
|
+
|
|
850
|
+
geos_data_on_ace_levels['age'] = ace_age
|
|
851
|
+
return geos_data_on_ace_levels
|
|
852
|
+
except Exception as err:
|
|
853
|
+
msg = f'Problem occurred in ACE profile group for dates {date_str} (GEOS file date {geos_dates[0]}): {err}'
|
|
854
|
+
raise err.__class__(msg)
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
def _bin_ace_to_geos_times(ace_dates):
|
|
858
|
+
"""
|
|
859
|
+
Group ACE profiles by which GEOS FP files surround them
|
|
860
|
+
|
|
861
|
+
:param ace_dates: an array of ACE profile datetimes
|
|
862
|
+
:type ace_dates: array of datetimes
|
|
863
|
+
|
|
864
|
+
:return: a dictionary where the keys will be Pandas timestamps representing the time of the GEOS file preceding the
|
|
865
|
+
profiles in that group and the values are the indices of the profiles in that group as a numpy array.
|
|
866
|
+
:rtype: dict
|
|
867
|
+
"""
|
|
868
|
+
def to_3hr(d):
|
|
869
|
+
return dt.datetime(d.year, d.month, d.day, to_3(d.hour))
|
|
870
|
+
|
|
871
|
+
def to_3(v):
|
|
872
|
+
return int(3*np.floor(v/3))
|
|
873
|
+
|
|
874
|
+
first_time = ace_dates.min()
|
|
875
|
+
last_time = ace_dates.max()
|
|
876
|
+
|
|
877
|
+
first_time = to_3hr(first_time)
|
|
878
|
+
last_time = to_3hr(last_time) + dt.timedelta(hours=3)
|
|
879
|
+
|
|
880
|
+
# Create the bins as floats so we can use numpy.digitize
|
|
881
|
+
ace_datenums = np.array([sat_utils.datetime2datenum(d) for d in ace_dates])
|
|
882
|
+
bin_edges = pd.date_range(first_time, last_time, freq='3H')
|
|
883
|
+
bin_edge_datenums = np.array([sat_utils.datetime2datenum(d) for d in bin_edges])
|
|
884
|
+
|
|
885
|
+
inds = np.digitize(ace_datenums, bin_edge_datenums) - 1
|
|
886
|
+
date_groups = OrderedDict()
|
|
887
|
+
for i in np.unique(inds):
|
|
888
|
+
date_groups[bin_edges[i]] = np.flatnonzero(inds == i)
|
|
889
|
+
|
|
890
|
+
return date_groups
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def _interp_geos_vars_to_ace_lat_lon(geos_file_path, geos_vars, ace_lon, ace_lat):
|
|
894
|
+
def interp_prof_helper(data, glat, glon, alat, alon, interp_inds):
|
|
895
|
+
profs_out = np.full([alat.size, data.shape[0]], np.nan)
|
|
896
|
+
for i, data_layer in enumerate(data):
|
|
897
|
+
# Reshape to nprofs-by-nlevels to be consistent with ACE
|
|
898
|
+
profs_out[:, i] = mm.lat_lon_interp(data_layer, glat, glon, alat, alon, interp_inds)
|
|
899
|
+
|
|
900
|
+
return profs_out
|
|
901
|
+
|
|
902
|
+
geos_data = dict()
|
|
903
|
+
geos_interp_inds = []
|
|
904
|
+
with ncdf.Dataset(geos_file_path) as nh:
|
|
905
|
+
geos_lon_half_res = 0.5 * float(nh.LongitudeResolution)
|
|
906
|
+
geos_lat_half_res = 0.5 * float(nh.LatitudeResolution)
|
|
907
|
+
|
|
908
|
+
for varname, scale in geos_vars.items():
|
|
909
|
+
# convert to 3D - remove singleton time dimension
|
|
910
|
+
geos_data[varname] = nh.variables[varname][0].filled(np.nan) * scale
|
|
911
|
+
|
|
912
|
+
for lon, lat in zip(ace_lon, ace_lat):
|
|
913
|
+
geos_interp_inds.append(mm.querry_indices(nh, lat, lon, geos_lat_half_res, geos_lon_half_res))
|
|
914
|
+
|
|
915
|
+
geos_lat = nh.variables['lat'][:].filled(np.nan)
|
|
916
|
+
geos_lon = nh.variables['lon'][:].filled(np.nan)
|
|
917
|
+
|
|
918
|
+
for varname, array in geos_data.items():
|
|
919
|
+
geos_data[varname] = interp_prof_helper(array, geos_lat, geos_lon, ace_lat, ace_lon, geos_interp_inds)
|
|
920
|
+
|
|
921
|
+
return geos_data
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def _interp_geos_vars_to_ace_times(ace_datetimes, geos_datetimes, geos_vars):
|
|
925
|
+
interped_vars = dict()
|
|
926
|
+
for ace_dt in ace_datetimes:
|
|
927
|
+
weight = sat_utils.time_weight_from_datetime(ace_dt, geos_datetimes[0], geos_datetimes[1])
|
|
928
|
+
for k in geos_vars[0].keys():
|
|
929
|
+
interped_vars[k] = weight * geos_vars[0][k] + (1 - weight) * geos_vars[1][k]
|
|
930
|
+
|
|
931
|
+
return interped_vars
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def save_ace_age_file(ace_out_file, vars_to_copy, orig_ace_file, eqlat, age):
|
|
935
|
+
def get_var_attrs(nh, varname):
|
|
936
|
+
return {k: nh.variables[varname].getncattr(k) for k in nh.variables[varname].ncattrs()}
|
|
937
|
+
|
|
938
|
+
with ncdf.Dataset(orig_ace_file, 'r') as nhread, ncdf.Dataset(ace_out_file, 'w') as nhwrite:
|
|
939
|
+
# Copy all the dimensions
|
|
940
|
+
for dimname in nhread.dimensions.keys():
|
|
941
|
+
try:
|
|
942
|
+
attributes = get_var_attrs(nhread, dimname)
|
|
943
|
+
ioutils.make_ncdim_helper(nhwrite, dimname, nhread.variables[dimname][:], **attributes)
|
|
944
|
+
except KeyError:
|
|
945
|
+
# Sometimes we get a dimension with no corresponding variable
|
|
946
|
+
nhwrite.createDimension(dimname, nhread.dimensions[dimname].size)
|
|
947
|
+
|
|
948
|
+
# Copy all the requested variables
|
|
949
|
+
for varname in vars_to_copy:
|
|
950
|
+
attributes = get_var_attrs(nhread, varname)
|
|
951
|
+
dims = nhread.variables[varname].dimensions
|
|
952
|
+
ioutils.make_ncvar_helper(nhwrite, varname, nhread.variables[varname][:], dims, **attributes)
|
|
953
|
+
|
|
954
|
+
# Add the new variables
|
|
955
|
+
dims = ('index', 'altitude')
|
|
956
|
+
ioutils.make_ncvar_helper(nhwrite, 'eqlat', eqlat, dims, long_name='equivalent_latitude', units='degrees_north',
|
|
957
|
+
description='Equivalent latitude computed from potential vorticity')
|
|
958
|
+
ioutils.make_ncvar_helper(nhwrite, 'age', age, dims, long_name='clams_age', units='years',
|
|
959
|
+
description='Time in years since the air entered the stratosphere')
|
|
960
|
+
|
|
961
|
+
ioutils.add_creation_info(nhwrite, creation_note='ace_fts_analysis.generate_ace_age_file')
|
|
962
|
+
nhwrite.setncattr('template_ace_file', orig_ace_file)
|
|
963
|
+
ioutils.add_dependent_file_hash(nhwrite, 'template_ace_sha1', orig_ace_file)
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def strat_co(ch4, T, P, age, co_0, oh=2.5e5, gamma=3):
|
|
967
|
+
R = 8.314 * 100 ** 3 / 100 / 6.626e23
|
|
968
|
+
|
|
969
|
+
def ndens_air(T, P):
|
|
970
|
+
return P / (R * T)
|
|
971
|
+
|
|
972
|
+
def kCO_OH(nair):
|
|
973
|
+
return 1.57e-13 + 3.54e-33 * nair
|
|
974
|
+
|
|
975
|
+
def kCH4_OH(T):
|
|
976
|
+
return 2.8e-14 * T ** 0.667 * np.exp(-1575 / T)
|
|
977
|
+
|
|
978
|
+
ndens = ndens_air(T, P)
|
|
979
|
+
age = age * 365.25 * 24 * 3600
|
|
980
|
+
ss = gamma * kCH4_OH(T) * ch4 / kCO_OH(ndens)
|
|
981
|
+
bc = co_0 * np.exp(-kCO_OH(ndens) * oh * age)
|
|
982
|
+
return ss + bc
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def make_cmam_excess_co_lut(save_file, cmam_file):
|
|
986
|
+
|
|
987
|
+
# First we need to rearrange the CMAM data so that year and month are organized separately.
|
|
988
|
+
with xr.open_dataset(cmam_file) as ds:
|
|
989
|
+
month_slices = [ds['vmrco'][i::12] for i in range(12)]
|
|
990
|
+
for i, month in enumerate(month_slices):
|
|
991
|
+
doy = mod_utils.day_of_year(pd.Timestamp(2005, i+1, 15))
|
|
992
|
+
mdim = OrderedDict([('doy', [doy])])
|
|
993
|
+
month = month.expand_dims(mdim, axis=1)
|
|
994
|
+
month.coords['time'] = [t.item().year for t in month.coords['time']]
|
|
995
|
+
month_slices[i] = month
|
|
996
|
+
|
|
997
|
+
# Duplicate the first and last months on the opposite ends to make periodic interpolation possible
|
|
998
|
+
first_month = month_slices[0].copy()
|
|
999
|
+
first_month.coords['doy'] = first_month.coords['doy'] + mod_utils.days_per_year
|
|
1000
|
+
|
|
1001
|
+
last_month = month_slices[-1].copy()
|
|
1002
|
+
last_month.coords['doy'] = last_month.coords['doy'] - mod_utils.days_per_year
|
|
1003
|
+
|
|
1004
|
+
month_slices.append(first_month)
|
|
1005
|
+
month_slices.insert(0, last_month)
|
|
1006
|
+
|
|
1007
|
+
co_monthly = xr.concat(month_slices, dim='doy')
|
|
1008
|
+
|
|
1009
|
+
# Limit it to after 2000 - there is a trend in CO but it looks like it more or less levels off after 2000.
|
|
1010
|
+
co_attrs = co_monthly.attrs
|
|
1011
|
+
co_monthly = co_monthly.sel(time=slice(2000, None))
|
|
1012
|
+
|
|
1013
|
+
# Then we just need to average in time and longitude to get monthly pressure-latitude slices.
|
|
1014
|
+
co_monthly = co_monthly.mean(dim='time').mean(dim='lon')
|
|
1015
|
+
co_monthly.attrs = co_attrs.copy()
|
|
1016
|
+
# will convert to ppb later, more convenient to leave in mole fraction for now because we need to calculate number
|
|
1017
|
+
# density too.
|
|
1018
|
+
|
|
1019
|
+
# Next we need to calculate the number density. Since we only have pressure from the model, we'll get temperature
|
|
1020
|
+
# from the US standard atmosphere.
|
|
1021
|
+
co_monthly.coords['plev'] /= 100 # convert Pa to hPa
|
|
1022
|
+
co_monthly.coords['plev'].attrs['units'] = 'hPa'
|
|
1023
|
+
|
|
1024
|
+
p = co_monthly.coords['plev']
|
|
1025
|
+
t, z = mod_utils.get_ussa_for_pres(p)
|
|
1026
|
+
t = xr.DataArray(t, coords=p.coords)
|
|
1027
|
+
z = xr.DataArray(z, coords=p.coords)
|
|
1028
|
+
|
|
1029
|
+
t.attrs['units'] = 'K'
|
|
1030
|
+
t.attrs['standard_name'] = 'absolute_temperature'
|
|
1031
|
+
t.attrs['long_name'] = 'temperature'
|
|
1032
|
+
|
|
1033
|
+
z.attrs['units'] = 'km'
|
|
1034
|
+
z.attrs['standard_name'] = 'altitude_above_sea_level'
|
|
1035
|
+
z.attrs['long_name'] = 'altitude'
|
|
1036
|
+
|
|
1037
|
+
nair = mod_utils.number_density_air(p, t)
|
|
1038
|
+
nair.attrs['units'] = 'molec. cm^{-3}'
|
|
1039
|
+
nair.attrs['standard_name'] = 'dry_number_density'
|
|
1040
|
+
nair.attrs['long_name'] = 'number density of air'
|
|
1041
|
+
|
|
1042
|
+
co_monthly_nd = co_monthly * nair
|
|
1043
|
+
co_monthly_nd.attrs['units'] = 'molec. cm^{-3}'
|
|
1044
|
+
co_monthly_nd.attrs['standard_name'] = 'co_number_density'
|
|
1045
|
+
co_monthly_nd.attrs['long_name'] = 'CO number density'
|
|
1046
|
+
|
|
1047
|
+
co_monthly *= 1e9
|
|
1048
|
+
co_monthly.attrs['units'] = 'ppb'
|
|
1049
|
+
|
|
1050
|
+
full_ds = xr.Dataset({'co': co_monthly, 'co_nd': co_monthly_nd, 'temperature': t, 'altitude': z, 'nair': nair})
|
|
1051
|
+
full_ds.attrs['cmam_file'] = cmam_file if not os.path.islink(cmam_file) else os.readlink(cmam_file)
|
|
1052
|
+
full_ds.attrs['cmam_sha1'] = ioutils.make_dependent_file_hash(cmam_file)
|
|
1053
|
+
full_ds.attrs['history'] = ioutils.make_creation_info('ace_fts_analysis.make_cmam_excess_co_lut')
|
|
1054
|
+
full_ds.to_netcdf(save_file)
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def make_excess_co_lut(save_file, ace_co_file, ace_ch4_file, ace_age_file, lat_type='eq', min_req_pts=10, smoothing_window_width=5):
|
|
1058
|
+
R = 8.314 * 100 ** 3 / 100 / 6.626e23
|
|
1059
|
+
flag_bits = {'clip': 2 ** 0, 'toofew': 2 ** 1, 'filled': 2 ** 2, 'ussa_any': 2**3, 'ussa_half': 2**4,
|
|
1060
|
+
'lowval': 2**5}
|
|
1061
|
+
flag_units = 'Bit meanings (least significant first): 1 = value <0 clipped, 2 = fewer than {} points, ' \
|
|
1062
|
+
'3 = NaN filled, 4 = At least one point used USSA P/T, 5 = >50% points used USSA P/T, ' \
|
|
1063
|
+
'6 = value below rolling mean - 1 sigma filled'.format(min_req_pts)
|
|
1064
|
+
|
|
1065
|
+
def ndens_air(T, P):
|
|
1066
|
+
return P / (R * T)
|
|
1067
|
+
|
|
1068
|
+
def kCO_OH(nair):
|
|
1069
|
+
return 1.57e-13 + 3.54e-33 * nair
|
|
1070
|
+
|
|
1071
|
+
def kCH4_OH(T):
|
|
1072
|
+
return 2.8e-14 * T ** 0.667 * np.exp(-1575 / T)
|
|
1073
|
+
|
|
1074
|
+
def strat_co(ch4, T, P, age, co_0, oh=2.5e5, gamma=3):
|
|
1075
|
+
ndens = ndens_air(T, P)
|
|
1076
|
+
age = age * 365.25 * 24 * 3600
|
|
1077
|
+
ss = gamma * kCH4_OH(T) * ch4 / kCO_OH(ndens)
|
|
1078
|
+
bc = co_0 * np.exp(-kCO_OH(ndens) * oh * age)
|
|
1079
|
+
return ss + bc
|
|
1080
|
+
|
|
1081
|
+
def fill_nans(ds, flags, primary_keys, dim_interp_order):
|
|
1082
|
+
# We want to interpolate/extrapolate along theta first. I tried originally doing periodic interpolation along
|
|
1083
|
+
# the day-of-year axis, but there's not enough data in the northen hemisphere polar summer to get the decrease
|
|
1084
|
+
# right. However, there are some columns that have 0 or 1 non-NaN values which the interpolator then chokes
|
|
1085
|
+
# on, so we need to find those columns and replace the NaNs with a fill value temporarily to avoid trying
|
|
1086
|
+
# to handle them here.
|
|
1087
|
+
was_nans = np.zeros(ds[primary_keys[0]].shape, dtype=np.bool_)
|
|
1088
|
+
for k in primary_keys:
|
|
1089
|
+
was_nans |= np.isnan(ds[k].data)
|
|
1090
|
+
|
|
1091
|
+
all_nans = dict()
|
|
1092
|
+
for dim, do_extrap in dim_interp_order.items():
|
|
1093
|
+
logger.debug('Filling NaNs along'.format(dim))
|
|
1094
|
+
|
|
1095
|
+
fill_val = 'extrapolate' if do_extrap else np.nan
|
|
1096
|
+
|
|
1097
|
+
for key, variable in ds.items():
|
|
1098
|
+
var_not_nans = ~np.isnan(variable.data)
|
|
1099
|
+
|
|
1100
|
+
# DataArray's sum function doesn't work with keepdims in v0.12.1. This is fixed in v0.12.2 which doesn't
|
|
1101
|
+
# seem to yet be available via conda
|
|
1102
|
+
dim_ind = variable.dims.index(dim)
|
|
1103
|
+
this_most_nans = var_not_nans.sum(axis=dim_ind, keepdims=True) < 2
|
|
1104
|
+
this_most_nans = np.broadcast_to(this_most_nans, variable.shape)
|
|
1105
|
+
# only replace NaNs in lines with 0 or 1 non-NaN values. Leave non-NaN values in those columns alone,
|
|
1106
|
+
# and leave NaNs elsewhere alone.
|
|
1107
|
+
this_most_nans = this_most_nans & ~var_not_nans
|
|
1108
|
+
variable.data[this_most_nans] = -999
|
|
1109
|
+
all_nans[key] = this_most_nans
|
|
1110
|
+
|
|
1111
|
+
ds = ds.interpolate_na(dim=dim, method='linear', fill_value=fill_val)
|
|
1112
|
+
|
|
1113
|
+
for key, this_most_nans in all_nans.items():
|
|
1114
|
+
ds[key].data[this_most_nans] = np.nan
|
|
1115
|
+
|
|
1116
|
+
nans_filled = np.zeros_like(was_nans)
|
|
1117
|
+
for k in primary_keys:
|
|
1118
|
+
nans_filled |= (was_nans & ~np.isnan(ds[k].data))
|
|
1119
|
+
|
|
1120
|
+
flags[nans_filled] = np.bitwise_or(flags[nans_filled], flag_bits['filled'])
|
|
1121
|
+
|
|
1122
|
+
return ds, flags
|
|
1123
|
+
|
|
1124
|
+
def bin_co(df_dict, all_bins, all_coords, bin_names, primary_key):
|
|
1125
|
+
bin_cols = tuple(df_dict.keys())
|
|
1126
|
+
coord_cols = []
|
|
1127
|
+
total_bins = 1
|
|
1128
|
+
for i, (coord, bin) in enumerate(zip(all_coords, all_bins)):
|
|
1129
|
+
total_bins *= (bin.size - 1)
|
|
1130
|
+
inds = np.digitize(coord.flatten(), bin) - 1
|
|
1131
|
+
colname = 'ind{}'.format(i)
|
|
1132
|
+
df_dict[colname] = inds
|
|
1133
|
+
coord_cols.append(colname)
|
|
1134
|
+
|
|
1135
|
+
df = pd.DataFrame(df_dict)
|
|
1136
|
+
all_bin_centers = [0.5 * (b[:-1] + b[1:]) for b in all_bins]
|
|
1137
|
+
|
|
1138
|
+
proto_array = np.full([np.size(b) - 1 for b in all_bins],
|
|
1139
|
+
np.nan if np.issubdtype(excess_co.dtype, np.floating) else 0)
|
|
1140
|
+
proto_array = xr.DataArray(proto_array, coords=all_bin_centers, dims=bin_names)
|
|
1141
|
+
means = xr.Dataset({k: proto_array.copy() for k in bin_cols})
|
|
1142
|
+
stds = xr.Dataset({k: proto_array.copy() for k in bin_cols})
|
|
1143
|
+
|
|
1144
|
+
flags = np.zeros(proto_array.shape, dtype=int)
|
|
1145
|
+
counts = np.zeros(proto_array.shape, dtype=int)
|
|
1146
|
+
|
|
1147
|
+
pbar = mod_utils.ProgressBar(total_bins, prefix='Binning', style='counter')
|
|
1148
|
+
iind = 0
|
|
1149
|
+
for inds, grp in df.groupby(coord_cols):
|
|
1150
|
+
pbar.print_bar(iind)
|
|
1151
|
+
iind += 1
|
|
1152
|
+
|
|
1153
|
+
if any(i < 0 for i in inds):
|
|
1154
|
+
# negative index indicates that the value is below the minimum bin so we skip it
|
|
1155
|
+
continue
|
|
1156
|
+
try:
|
|
1157
|
+
subdata = grp[primary_key]
|
|
1158
|
+
counts[inds] = (~subdata.isna()).sum()
|
|
1159
|
+
if subdata.size < min_req_pts:
|
|
1160
|
+
flags[inds] = np.bitwise_or(flags[inds], flag_bits['toofew'])
|
|
1161
|
+
continue
|
|
1162
|
+
|
|
1163
|
+
for k in bin_cols:
|
|
1164
|
+
subdata = grp[k]
|
|
1165
|
+
means[k][inds] = np.nanmean(subdata)
|
|
1166
|
+
stds[k][inds] = np.nanstd(subdata)
|
|
1167
|
+
|
|
1168
|
+
except IndexError:
|
|
1169
|
+
# inds will be too large if any of the values is outside the right edge of the last bin, so just skip
|
|
1170
|
+
# those since we don't want to bin them. We caught any outside the left edge of the bin checking if any
|
|
1171
|
+
# are < 0
|
|
1172
|
+
continue
|
|
1173
|
+
|
|
1174
|
+
pbar.finish()
|
|
1175
|
+
|
|
1176
|
+
return means, stds, flags, counts, all_bin_centers
|
|
1177
|
+
|
|
1178
|
+
def replace_low_values(darray, dim, flags, remove_neg=True, extra_mask=None):
|
|
1179
|
+
darray = darray.copy()
|
|
1180
|
+
periodic = dim == 'doy'
|
|
1181
|
+
# Concatenate extra slices to allow periodic interpolation if operating along DOY
|
|
1182
|
+
if periodic:
|
|
1183
|
+
ex_width = int(np.ceil(smoothing_window_width/2))
|
|
1184
|
+
dim_max = darray.coords[dim].max()
|
|
1185
|
+
darray_start = darray.isel(**{dim: slice(None, ex_width)})
|
|
1186
|
+
darray_start.coords[dim] = darray_start.coords[dim] + dim_max
|
|
1187
|
+
darray_end = darray.isel(**{dim: slice(-ex_width, None)})
|
|
1188
|
+
darray_end.coords[dim] = darray_end.coords[dim] - dim_max
|
|
1189
|
+
darray = xr.concat([darray_end, darray, darray_start], dim=dim)
|
|
1190
|
+
|
|
1191
|
+
if remove_neg:
|
|
1192
|
+
darray.data[darray.data < 0] = np.nan
|
|
1193
|
+
kwargs = {dim: smoothing_window_width, 'center': True}
|
|
1194
|
+
mean = darray.rolling(**kwargs).mean()
|
|
1195
|
+
std = darray.rolling(**kwargs).std()
|
|
1196
|
+
xx = darray < (mean - std)
|
|
1197
|
+
if extra_mask is not None:
|
|
1198
|
+
xx = xx | extra_mask
|
|
1199
|
+
darray.data[xx.data] = np.nan
|
|
1200
|
+
|
|
1201
|
+
darray = darray.interpolate_na(dim=dim)
|
|
1202
|
+
if periodic:
|
|
1203
|
+
s = slice(ex_width, -ex_width)
|
|
1204
|
+
darray = darray.isel(**{dim: s})
|
|
1205
|
+
xx = xx.isel(**{dim: s})
|
|
1206
|
+
flags[xx.data] = np.bitwise_or(flags[xx.data], flag_bits['lowval'])
|
|
1207
|
+
return darray, flags
|
|
1208
|
+
|
|
1209
|
+
#################
|
|
1210
|
+
# Load ACE data #
|
|
1211
|
+
#################
|
|
1212
|
+
|
|
1213
|
+
with ncdf.Dataset(ace_age_file) as ds:
|
|
1214
|
+
ace_dates = read_ace_date(ds)
|
|
1215
|
+
ace_alt = read_ace_var(ds, 'altitude', None)
|
|
1216
|
+
qflags = read_ace_var(ds, 'quality_flag', None)
|
|
1217
|
+
|
|
1218
|
+
if lat_type == 'geog':
|
|
1219
|
+
ace_lat = read_ace_var(ds, 'latitude', None)
|
|
1220
|
+
ace_lat = np.broadcast_to(ace_lat.reshape(-1, 1), qflags.shape)
|
|
1221
|
+
elif lat_type == 'eq':
|
|
1222
|
+
ace_lat = read_ace_var(ds, 'eqlat', None)
|
|
1223
|
+
# Equivalent latitude may not be defined on all levels. Fill in any internal levels with linear
|
|
1224
|
+
# interpolation then extends the profile assuming that the equivalent latitude is constant above the top
|
|
1225
|
+
# level.
|
|
1226
|
+
ace_lat = xr.DataArray(ace_lat, coords=[np.arange(ace_lat.shape[0]), ace_alt], dims=['index', 'alt'])
|
|
1227
|
+
ace_lat = ace_lat.interpolate_na(dim='alt', method='linear').interpolate_na(dim='alt', method='nearest', fill_value='extrapolate')
|
|
1228
|
+
ace_lat = ace_lat.data
|
|
1229
|
+
else:
|
|
1230
|
+
raise ValueError('lat must be "geog" or "eq"')
|
|
1231
|
+
|
|
1232
|
+
ace_t = read_ace_var(ds, 'temperature', qflags)
|
|
1233
|
+
ace_p = read_ace_var(ds, 'pressure', qflags) * 1013.25 # atm -> hPa
|
|
1234
|
+
ace_pt = read_ace_theta(ds, qflags=qflags)
|
|
1235
|
+
ace_age = read_ace_var(ds, 'age', qflags)
|
|
1236
|
+
|
|
1237
|
+
with ncdf.Dataset(ace_ch4_file) as ds:
|
|
1238
|
+
qflags = read_ace_var(ds, 'quality_flag', None)
|
|
1239
|
+
ace_ch4 = read_ace_var(ds, 'CH4', qflags)
|
|
1240
|
+
|
|
1241
|
+
with ncdf.Dataset(ace_co_file) as ds:
|
|
1242
|
+
qflags = read_ace_var(ds, 'quality_flag', None)
|
|
1243
|
+
all_ace_co = read_ace_var(ds, 'CO', qflags)
|
|
1244
|
+
|
|
1245
|
+
ace_doys = np.array([mod_utils.day_of_year(d) for d in ace_dates])
|
|
1246
|
+
ace_doys = np.broadcast_to(ace_doys.reshape(-1, 1), all_ace_co.shape)
|
|
1247
|
+
|
|
1248
|
+
# Create versions of T and P filled in by the US standard atmosphere for the mesospheric CO profile table.
|
|
1249
|
+
xx_ussa = np.isnan(ace_t) | np.isnan(ace_p)
|
|
1250
|
+
ace_alt_full = np.broadcast_to(ace_alt.reshape(1, -1), ace_t.shape)
|
|
1251
|
+
ace_t_ussa = ace_t.copy()
|
|
1252
|
+
ace_p_ussa = ace_p.copy()
|
|
1253
|
+
ace_t_ussa[xx_ussa], ace_p_ussa[xx_ussa] = mod_utils.get_ussa_for_alts(ace_alt_full[xx_ussa])
|
|
1254
|
+
|
|
1255
|
+
co_calc = strat_co(ace_ch4, ace_t, ace_p, ace_age, 50.0e-9)
|
|
1256
|
+
excess_co = (all_ace_co - co_calc)*1e9 # calculate excess and convert to ppb
|
|
1257
|
+
excess_notnans = ~np.isnan(excess_co)
|
|
1258
|
+
excess_co = excess_co[excess_notnans]
|
|
1259
|
+
|
|
1260
|
+
air_nd = mod_utils.number_density_air(ace_p_ussa, ace_t_ussa)
|
|
1261
|
+
ace_co_nd = all_ace_co * air_nd
|
|
1262
|
+
not_all_nans = np.any(~np.isnan(ace_co_nd), axis=0)
|
|
1263
|
+
air_nd = air_nd[:, not_all_nans]
|
|
1264
|
+
ace_co_nd = ace_co_nd[:, not_all_nans]
|
|
1265
|
+
ace_p_ussa = ace_p_ussa[:, not_all_nans]
|
|
1266
|
+
ace_t_ussa = ace_t_ussa[:, not_all_nans]
|
|
1267
|
+
xx_ussa = xx_ussa[:, not_all_nans]
|
|
1268
|
+
ace_alt_full = ace_alt_full[:, not_all_nans]
|
|
1269
|
+
|
|
1270
|
+
meso_notnans = ~np.isnan(ace_co_nd)
|
|
1271
|
+
ace_co_nd = ace_co_nd[meso_notnans]
|
|
1272
|
+
ace_p_ussa = ace_p_ussa[meso_notnans]
|
|
1273
|
+
ace_t_ussa = ace_t_ussa[meso_notnans]
|
|
1274
|
+
air_nd = air_nd[meso_notnans]
|
|
1275
|
+
xx_ussa = xx_ussa[meso_notnans]
|
|
1276
|
+
|
|
1277
|
+
lat_bins = np.arange(-90, 91, 10)
|
|
1278
|
+
doy_bins = np.arange(0, 370, 5)
|
|
1279
|
+
doy_bins[-1] = 366 # account for leap years
|
|
1280
|
+
theta_bins = np.arange(350, 4150, 100)
|
|
1281
|
+
alt_bins = np.arange(ace_alt[not_all_nans].min(), ace_alt[not_all_nans].max()+5, 5.)
|
|
1282
|
+
|
|
1283
|
+
excess_bins = [lat_bins, doy_bins, theta_bins]
|
|
1284
|
+
excess_coords = [ace_lat[excess_notnans], ace_doys[excess_notnans], ace_pt[excess_notnans]]
|
|
1285
|
+
excess_bin_names = ['lat', 'doy', 'theta']
|
|
1286
|
+
excess_bin_units = ['degrees_north', 'day of year (0-based)', 'K']
|
|
1287
|
+
|
|
1288
|
+
meso_bins = [lat_bins, doy_bins, alt_bins]
|
|
1289
|
+
meso_coords = [ace_lat[:, not_all_nans][meso_notnans], ace_doys[:, not_all_nans][meso_notnans], ace_alt_full[meso_notnans]]
|
|
1290
|
+
meso_bin_names = ['lat', 'doy', 'altitude']
|
|
1291
|
+
meso_bin_units = ['degrees_north', 'doy of year (0-based)', 'km']
|
|
1292
|
+
|
|
1293
|
+
####################################################
|
|
1294
|
+
# Use pandas groupby() to bin the data efficiently #
|
|
1295
|
+
####################################################
|
|
1296
|
+
|
|
1297
|
+
excess_means, excess_stds, excess_flags, excess_counts, excess_bin_centers = bin_co(
|
|
1298
|
+
{'excess_co': excess_co}, all_bins=excess_bins, all_coords=excess_coords, bin_names=excess_bin_names,
|
|
1299
|
+
primary_key='excess_co'
|
|
1300
|
+
)
|
|
1301
|
+
excess_means['excess_co'].attrs['unit'] = 'ppb'
|
|
1302
|
+
excess_means['excess_co'].attrs['description'] = 'Excess CO in ACE-FTS data beyond that predicted by a kinetic ' \
|
|
1303
|
+
'model of the CO chemistry in the stratosphere.'
|
|
1304
|
+
excess_stds['excess_co'].attrs['unit'] = 'ppb'
|
|
1305
|
+
excess_stds['excess_co'].attrs['description'] = 'Standard deviation of the excess CO in this bin.'
|
|
1306
|
+
|
|
1307
|
+
meso_means, meso_stds, meso_flags, meso_counts, meso_bin_centers = bin_co(
|
|
1308
|
+
{'co_nd': ace_co_nd, 'air_nd': air_nd, 'pressure': ace_p_ussa, 'temperature': ace_t_ussa, 'is_ussa': xx_ussa},
|
|
1309
|
+
all_bins=meso_bins, all_coords=meso_coords, bin_names=meso_bin_names, primary_key='co_nd'
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
inds = meso_means['is_ussa'].data > 0
|
|
1313
|
+
meso_flags[inds] = np.bitwise_or(meso_flags[inds], flag_bits['ussa_any'])
|
|
1314
|
+
inds = meso_means['is_ussa'].data > 0.5
|
|
1315
|
+
meso_flags[inds] = np.bitwise_or(meso_flags[inds], flag_bits['ussa_half'])
|
|
1316
|
+
|
|
1317
|
+
meso_means = meso_means.drop('is_ussa')
|
|
1318
|
+
meso_stds = meso_stds.drop('is_ussa')
|
|
1319
|
+
|
|
1320
|
+
meso_means['co_nd'].attrs['unit'] = 'molec cm^{-3}'
|
|
1321
|
+
meso_means['co_nd'].attrs['description'] = 'ACE CO in number density. ACE T and P extended using the US standard ' \
|
|
1322
|
+
'atmosphere as needed.'
|
|
1323
|
+
meso_means['air_nd'].attrs['unit'] = 'molec cm^{-3}'
|
|
1324
|
+
meso_means['air_nd'].attrs['description'] = 'Number density of air calculated using the ideal gas law with ACE T ' \
|
|
1325
|
+
'and P extended using the US standard atmosphere as needed'
|
|
1326
|
+
meso_means['pressure'].attrs['unit'] = 'hPa'
|
|
1327
|
+
meso_means['pressure'].attrs['description'] = 'ACE pressure extended using the US standard atmosphere as needed'
|
|
1328
|
+
meso_means['temperature'].attrs['unit'] = 'K'
|
|
1329
|
+
meso_means['temperature'].attrs['description'] = 'ACE temperatures extended using the US standard atmosphere as needed'
|
|
1330
|
+
|
|
1331
|
+
meso_stds['co_nd'].attrs['unit'] = 'molec cm^{-3}'
|
|
1332
|
+
meso_stds['co_nd'].attrs['description'] = 'Standard deviation of ACE CO in number density in the bins.'
|
|
1333
|
+
meso_stds['air_nd'].attrs['unit'] = 'molec cm^{-3}'
|
|
1334
|
+
meso_stds['air_nd'].attrs['description'] = 'Standard deviation of number density of air in the bins'
|
|
1335
|
+
meso_stds['pressure'].attrs['unit'] = 'hPa'
|
|
1336
|
+
meso_stds['pressure'].attrs['description'] = 'Standard deviation of ACE pressure in the bins.'
|
|
1337
|
+
meso_stds['temperature'].attrs['unit'] = 'K'
|
|
1338
|
+
meso_stds['temperature'].attrs['description'] = 'Standard deviation of ACE temperatures in the bins'
|
|
1339
|
+
|
|
1340
|
+
all_bin_edges = excess_bins + meso_bins[-1:]
|
|
1341
|
+
all_bin_centers = excess_bin_centers + meso_bin_centers[-1:]
|
|
1342
|
+
all_bin_names = excess_bin_names + meso_bin_names[-1:]
|
|
1343
|
+
all_bin_units = excess_bin_units + meso_bin_units[-1:]
|
|
1344
|
+
|
|
1345
|
+
#######################################################################################
|
|
1346
|
+
# Fill in NaNs and clip to positive values. Track these actions in the flags variable #
|
|
1347
|
+
#######################################################################################
|
|
1348
|
+
inds = excess_means['excess_co'].data < 0
|
|
1349
|
+
excess_flags[inds] = np.bitwise_or(excess_flags[inds], flag_bits['clip'])
|
|
1350
|
+
excess_means['excess_co'] = excess_means['excess_co'].clip(0)
|
|
1351
|
+
|
|
1352
|
+
excess_interp_order = OrderedDict([('theta', True), ('lat', True), ('doy', False)])
|
|
1353
|
+
excess_means, excess_flags = fill_nans(excess_means, excess_flags, ['excess_co'], excess_interp_order)
|
|
1354
|
+
meso_interp_order = OrderedDict([('doy', False), ('lat', True), ('altitude', True)])
|
|
1355
|
+
meso_means, meso_flags = fill_nans(meso_means, meso_flags, ['co_nd', 'air_nd', 'pressure', 'temperature'],
|
|
1356
|
+
meso_interp_order)
|
|
1357
|
+
|
|
1358
|
+
excess_means['excess_co'], excess_flags = replace_low_values(excess_means['excess_co'], 'doy', excess_flags)
|
|
1359
|
+
meso_means['co_nd'], meso_flags = replace_low_values(meso_means['co_nd'], 'doy', meso_flags)
|
|
1360
|
+
|
|
1361
|
+
# Add the count variables
|
|
1362
|
+
excess_means['excess_co_counts'] = xr.DataArray(excess_counts, coords=excess_means['excess_co'].coords)
|
|
1363
|
+
excess_means['excess_co_counts'].attrs['unit'] = '#'
|
|
1364
|
+
excess_means['excess_co_counts'].attrs['description'] = 'Number of ACE points contributing to the bin'
|
|
1365
|
+
meso_means['co_nd_counts'] = xr.DataArray(meso_counts, coords=meso_means['co_nd'].coords)
|
|
1366
|
+
meso_means['co_nd_counts'].attrs['unit'] = '#'
|
|
1367
|
+
meso_means['co_nd_counts'].attrs['description'] = 'Number of ACE points contributing to the bin'
|
|
1368
|
+
|
|
1369
|
+
_save_excess_co_lut(save_file=save_file,
|
|
1370
|
+
theta_binned_means=excess_means, theta_binned_stds=excess_stds, theta_binned_flags=excess_flags,
|
|
1371
|
+
alt_binned_means=meso_means, alt_binned_stds=meso_stds, alt_binned_flags=meso_flags,
|
|
1372
|
+
bin_edges=all_bin_edges, bin_centers=all_bin_centers, bin_names=all_bin_names,
|
|
1373
|
+
bin_units=all_bin_units, theta_binned_dims=excess_bin_names, alt_binned_dims=meso_bin_names,
|
|
1374
|
+
flag_description=flag_units, lat_type=lat_type,
|
|
1375
|
+
ace_co_file=ace_co_file, ace_ch4_file=ace_ch4_file, ace_age_file=ace_age_file)
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def _save_excess_co_lut(save_file, theta_binned_means, theta_binned_stds, theta_binned_flags,
|
|
1379
|
+
alt_binned_means, alt_binned_stds, alt_binned_flags,
|
|
1380
|
+
bin_edges, bin_centers, bin_names, bin_units,
|
|
1381
|
+
flag_description, theta_binned_dims, alt_binned_dims, lat_type,
|
|
1382
|
+
ace_co_file, ace_ch4_file, ace_age_file):
|
|
1383
|
+
|
|
1384
|
+
with ncdf.Dataset(save_file, 'w') as nh:
|
|
1385
|
+
|
|
1386
|
+
dims = []
|
|
1387
|
+
for name, centers, edges, units in zip(bin_names, bin_centers, bin_edges, bin_units):
|
|
1388
|
+
this_dim = ioutils.make_ncdim_helper(nh, name, centers, unit=units)
|
|
1389
|
+
dims.append(this_dim)
|
|
1390
|
+
ioutils.make_ncdim_helper(nh, name + '_bin_edges', edges, unit=units)
|
|
1391
|
+
|
|
1392
|
+
for k in theta_binned_means.keys():
|
|
1393
|
+
data = theta_binned_means[k]
|
|
1394
|
+
ioutils.make_ncvar_helper(nh, k, data.data, dims=theta_binned_dims,
|
|
1395
|
+
flag_field='theta_binned_flags', **data.attrs)
|
|
1396
|
+
if k in theta_binned_stds:
|
|
1397
|
+
std = theta_binned_stds[k]
|
|
1398
|
+
ioutils.make_ncvar_helper(nh, k+'_std', std.data, dims=theta_binned_dims,
|
|
1399
|
+
flag_field='theta_binned_flags', **std.attrs)
|
|
1400
|
+
ioutils.make_ncvar_helper(nh, 'theta_binned_flags', theta_binned_flags, dims=theta_binned_dims,
|
|
1401
|
+
unit='Bit array flag', description=flag_description)
|
|
1402
|
+
|
|
1403
|
+
for k in alt_binned_means.keys():
|
|
1404
|
+
data = alt_binned_means[k]
|
|
1405
|
+
ioutils.make_ncvar_helper(nh, k, data.data, dims=alt_binned_dims,
|
|
1406
|
+
flag_field='alt_binned_flags', **data.attrs)
|
|
1407
|
+
if k in alt_binned_stds:
|
|
1408
|
+
std = alt_binned_stds[k]
|
|
1409
|
+
ioutils.make_ncvar_helper(nh, k+'_std', std.data, dims=alt_binned_dims,
|
|
1410
|
+
flag_field='alt_binned_flags', **std.attrs)
|
|
1411
|
+
ioutils.make_ncvar_helper(nh, 'alt_binned_flags', alt_binned_flags, dims=alt_binned_dims,
|
|
1412
|
+
unit='Bit array flag', description=flag_description)
|
|
1413
|
+
|
|
1414
|
+
ioutils.add_dependent_file_hash(nh, 'ace_co_file', ace_co_file)
|
|
1415
|
+
ioutils.add_dependent_file_hash(nh, 'ace_ch4_file', ace_ch4_file)
|
|
1416
|
+
ioutils.add_dependent_file_hash(nh, 'ace_age_file', ace_age_file)
|
|
1417
|
+
ioutils.add_creation_info(nh, 'ace_fts_analysis.create_excess_co_lut')
|
|
1418
|
+
nh.setncattr('latitude_type', 'Binned by {} latitude'.format(lat_type))
|