ginput 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ginput/__init__.py +4 -0
- ginput/common_utils/__init__.py +0 -0
- ginput/common_utils/ggg_logging.py +62 -0
- ginput/common_utils/ioutils.py +304 -0
- ginput/common_utils/mod_constants.py +24 -0
- ginput/common_utils/mod_utils.py +2663 -0
- ginput/common_utils/readers.py +477 -0
- ginput/common_utils/run_utils.py +57 -0
- ginput/common_utils/sat_utils.py +67 -0
- ginput/common_utils/versioning.py +251 -0
- ginput/common_utils/writers.py +377 -0
- ginput/data/.gitignore +1 -0
- ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
- ginput/data/ML_monthly_obs_ch4.txt +498 -0
- ginput/data/ML_monthly_obs_co.txt +412 -0
- ginput/data/ML_monthly_obs_co2.txt +597 -0
- ginput/data/ML_monthly_obs_n2o.txt +330 -0
- ginput/data/SMO_monthly_obs_ch4.txt +499 -0
- ginput/data/SMO_monthly_obs_co.txt +416 -0
- ginput/data/SMO_monthly_obs_co2.txt +613 -0
- ginput/data/SMO_monthly_obs_n2o.txt +330 -0
- ginput/data/ace_fn2o_lut.nc +0 -0
- ginput/data/age_spectra/age.midlat.txt +47 -0
- ginput/data/age_spectra/age.tropics.txt +46 -0
- ginput/data/age_spectra/age.vortex.txt +31 -0
- ginput/data/age_spectra/agespec.midlat.txt +47 -0
- ginput/data/age_spectra/agespec.tropics.txt +46 -0
- ginput/data/age_spectra/agespec.vortex.txt +31 -0
- ginput/data/age_spectra/time.txt +3001 -0
- ginput/data/ch4_hf_slopes.nc +0 -0
- ginput/data/ch4_strat_lut.nc +0 -0
- ginput/data/clams_age_clim_scaled.nc +0 -0
- ginput/data/co2_strat_lut.nc +0 -0
- ginput/data/fo2_data/.gitignore +2 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
- ginput/data/hf_strat_lut.nc +0 -0
- ginput/data/isotopologs.dat +199 -0
- ginput/data/meso_co_lut.nc +0 -0
- ginput/data/n2o_ch4_acefts.nc +0 -0
- ginput/data/n2o_strat_lut.nc +0 -0
- ginput/data/o2_mean_dmf.dat +52 -0
- ginput/data/summer_35N.vmr +182 -0
- ginput/data/washenfelder03_table_s3.txt +83 -0
- ginput/download/__init__.py +0 -0
- ginput/download/download_utils.py +46 -0
- ginput/download/get_GEOS5.py +367 -0
- ginput/download/get_MERRA2.py +177 -0
- ginput/download/get_NOAA_flask_data.py +103 -0
- ginput/download/get_fo2_data.py +140 -0
- ginput/mod_maker/__init__.py +0 -0
- ginput/mod_maker/mod_maker.py +2470 -0
- ginput/mod_maker/slantify.py +450 -0
- ginput/mod_maker/tccon_sites.py +570 -0
- ginput/priors/__init__.py +0 -0
- ginput/priors/acos_interface.py +1376 -0
- ginput/priors/automation.py +313 -0
- ginput/priors/backend_analysis/__init__.py +0 -0
- ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
- ginput/priors/backend_analysis/ace_fts_val.py +452 -0
- ginput/priors/backend_analysis/acos_comparison.py +550 -0
- ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
- ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
- ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
- ginput/priors/backend_analysis/backend_driver.py +90 -0
- ginput/priors/backend_analysis/backend_utils.py +358 -0
- ginput/priors/backend_analysis/clams.py +71 -0
- ginput/priors/backend_analysis/create_test_priors.py +853 -0
- ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
- ginput/priors/backend_analysis/mls_val.py +217 -0
- ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
- ginput/priors/fo2_prep.py +311 -0
- ginput/priors/map_maker.py +193 -0
- ginput/priors/mlo_smo_prep.py +1146 -0
- ginput/priors/tccon_priors.py +3482 -0
- ginput-1.3.0.dist-info/LICENSE.txt +203 -0
- ginput-1.3.0.dist-info/METADATA +125 -0
- ginput-1.3.0.dist-info/RECORD +86 -0
- ginput-1.3.0.dist-info/WHEEL +5 -0
- ginput-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
from collections import OrderedDict
|
|
2
|
+
import datetime as dt
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import netCDF4 as ncdf
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from . import mod_utils
|
|
12
|
+
from .mod_utils import ModelError
|
|
13
|
+
from .versioning import GeosVersion, GeosSource
|
|
14
|
+
from .ggg_logging import logger
|
|
15
|
+
|
|
16
|
+
from typing import Dict, Union
|
|
17
|
+
|
|
18
|
+
def read_out_file(out_file, as_dataframes=False, replace_fills=False):
|
|
19
|
+
"""Read a GGG output file
|
|
20
|
+
|
|
21
|
+
This function can read most GGG output files, including .col files and post processing
|
|
22
|
+
text files (but not the .private.nc file). It expects a file where:
|
|
23
|
+
|
|
24
|
+
* the first line is a sequence of numbers, with the first one being the number of header
|
|
25
|
+
lines
|
|
26
|
+
* the last header line is a space-separated list of column names
|
|
27
|
+
* the rest of the file is a space-separated data table.
|
|
28
|
+
|
|
29
|
+
Parameters
|
|
30
|
+
----------
|
|
31
|
+
out_file : str
|
|
32
|
+
Path to the file to read
|
|
33
|
+
|
|
34
|
+
as_dataframes : bool
|
|
35
|
+
If `True`, then the output file is read in as a dataframe. If `False`, it is converted
|
|
36
|
+
to a dictionary when returning.
|
|
37
|
+
|
|
38
|
+
replace_fills : bool or float
|
|
39
|
+
If `False`, fill values are kept as is. If `True`, this will look for a line in the header
|
|
40
|
+
with the format `"missing: <number>`. Any number within floating point rounding of that
|
|
41
|
+
number are replaced with NaNs. If this is a number, then any values greater than or equal
|
|
42
|
+
to that number are treated as fills and replaced with NaNs. This last is the safest; GGG
|
|
43
|
+
post processing typically uses numbers >1e35 as "missing" values, so usually any value greater
|
|
44
|
+
than that can be replaced.
|
|
45
|
+
|
|
46
|
+
Note that `TypeError` occurences during comparison are ignored. This allows e.g. the "spectrum"
|
|
47
|
+
column to be skipped, but means that any formatting issues that cause a column to be read in
|
|
48
|
+
as an series of objects, rather than a numeric type, can cause that column to be skipped.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
pandas.DataFrame or dict
|
|
53
|
+
The contents of the file as a dataframe or dict, depending on `as_dataframes`.
|
|
54
|
+
"""
|
|
55
|
+
n_header_lines = mod_utils.get_num_header_lines(out_file)
|
|
56
|
+
df = pd.read_csv(out_file, header=n_header_lines-1, sep='\s+')
|
|
57
|
+
|
|
58
|
+
if replace_fills is True:
|
|
59
|
+
# Get the fill value from the header
|
|
60
|
+
with open(out_file) as f:
|
|
61
|
+
fill_value = None
|
|
62
|
+
for _ in range(n_header_lines):
|
|
63
|
+
line = f.readline()
|
|
64
|
+
if line.startswith('missing:'):
|
|
65
|
+
fill_value = float(line.split(':', 1)[1])
|
|
66
|
+
break
|
|
67
|
+
|
|
68
|
+
if fill_value is None:
|
|
69
|
+
raise IOError('Could not find fill value in the header of {}'.format(out_file))
|
|
70
|
+
|
|
71
|
+
is_fill = lambda val: np.isclose(val, fill_value)
|
|
72
|
+
|
|
73
|
+
elif replace_fills is not False:
|
|
74
|
+
fill_value = replace_fills
|
|
75
|
+
replace_fills = True
|
|
76
|
+
is_fill = lambda val: val >= fill_value
|
|
77
|
+
|
|
78
|
+
if replace_fills:
|
|
79
|
+
for colname, coldata in df.iteritems():
|
|
80
|
+
try:
|
|
81
|
+
xx_fills = is_fill(coldata)
|
|
82
|
+
except TypeError:
|
|
83
|
+
continue
|
|
84
|
+
else:
|
|
85
|
+
df.loc[xx_fills, colname] = np.nan
|
|
86
|
+
|
|
87
|
+
if not as_dataframes:
|
|
88
|
+
return df.to_dict()
|
|
89
|
+
else:
|
|
90
|
+
return df
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def read_mod_file(mod_file, as_dataframes=False):
|
|
94
|
+
"""
|
|
95
|
+
Read a TCCON .mod file.
|
|
96
|
+
|
|
97
|
+
:param mod_file: the path to the mod file.
|
|
98
|
+
:type mod_file: str
|
|
99
|
+
|
|
100
|
+
:param as_dataframes: if ``True``, then the collection of variables will be kept as dataframes. If ``False``
|
|
101
|
+
(default), they are converted to dictionaries of floats or numpy arrays.
|
|
102
|
+
:type as_dataframes: bool
|
|
103
|
+
|
|
104
|
+
:return: a dictionary with keys 'file' (values derived from file name), 'constants' (constant values stored in the
|
|
105
|
+
.mod file header), 'scalar' (values like surface height and tropopause pressure that are only defined once per
|
|
106
|
+
profile) and 'profile' (profile variables) containing the respective variables. These values will be dictionaries
|
|
107
|
+
or data frames, depending on ``as_dataframes``.
|
|
108
|
+
:rtype: dict
|
|
109
|
+
"""
|
|
110
|
+
n_header_lines = mod_utils.get_num_header_lines(mod_file)
|
|
111
|
+
# Read the constants from the second line of the file. There's no header for these, we just have to rely on the
|
|
112
|
+
# same constants being in the same position.
|
|
113
|
+
constant_vars = pd.read_csv(mod_file, sep=r'\s+', header=None, nrows=1, skiprows=1,
|
|
114
|
+
names=('earth_radius', 'ecc2', 'obs_lat', 'surface_gravity',
|
|
115
|
+
'profile_base_geometric_alt', 'base_pressure', 'tropopause_pressure'))
|
|
116
|
+
# Read the scalar variables (e.g. surface pressure, SZA, tropopause) first. We just have to assume their headers are
|
|
117
|
+
# on line 3 and values on line 4 of the file, the first number in the first line gives us the line the profile
|
|
118
|
+
# variables start on.
|
|
119
|
+
scalar_vars = pd.read_csv(mod_file, sep=r'\s+', header=2, nrows=1)
|
|
120
|
+
|
|
121
|
+
# Get the GEOS versions from the header. Also extract the CO source, since the priors
|
|
122
|
+
# code relies on that and we should support pre-version 1.2.1 files.
|
|
123
|
+
geos_versions = _read_mod_file_geos_sources(mod_file)
|
|
124
|
+
constant_vars['co_source'] = _read_mod_file_co_source(mod_file, geos_versions)
|
|
125
|
+
|
|
126
|
+
# Now read the profile vars.
|
|
127
|
+
profile_vars = pd.read_csv(mod_file, sep=r'\s+', header=n_header_lines-1)
|
|
128
|
+
|
|
129
|
+
# Also get the information that's only in the file name (namely date and longitude, we'll also read the latitude
|
|
130
|
+
# because it's there).
|
|
131
|
+
file_vars = dict()
|
|
132
|
+
base_name = os.path.basename(mod_file)
|
|
133
|
+
file_vars['datetime'] = mod_utils.find_datetime_substring(base_name, out_type=dt.datetime)
|
|
134
|
+
file_vars['lon'] = mod_utils.find_lon_substring(base_name, to_float=True)
|
|
135
|
+
file_vars['lat'] = mod_utils.find_lat_substring(base_name, to_float=True)
|
|
136
|
+
|
|
137
|
+
# Check that the header latitude and the file name latitude don't differ by more than 0.5 degree. Even if rounded
|
|
138
|
+
# to an integer for the file name, the difference should not exceed 0.5 degree.
|
|
139
|
+
lat_diff_threshold = 0.5
|
|
140
|
+
if np.abs(file_vars['lat'] - constant_vars['obs_lat'].item()) > lat_diff_threshold:
|
|
141
|
+
raise ModelError('The latitude in the file name and .mod file header differ by more than {lim} deg ({name} vs. '
|
|
142
|
+
'{head}). This indicates a possibly malformed .mod file.'
|
|
143
|
+
.format(lim=lat_diff_threshold, name=file_vars['lat'], head=constant_vars['obs_lat'].item())
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
out_dict = dict()
|
|
147
|
+
if as_dataframes:
|
|
148
|
+
out_dict['file'] = pd.DataFrame(file_vars, index=[0])
|
|
149
|
+
out_dict['constants'] = constant_vars
|
|
150
|
+
out_dict['scalar'] = scalar_vars
|
|
151
|
+
out_dict['profile'] = profile_vars
|
|
152
|
+
else:
|
|
153
|
+
out_dict['file'] = file_vars
|
|
154
|
+
out_dict['constants'] = {k: v.item() for k, v in constant_vars.items()}
|
|
155
|
+
out_dict['scalar'] = {k: v.item() for k, v in scalar_vars.items()}
|
|
156
|
+
out_dict['profile'] = {k: v.values for k, v in profile_vars.items()}
|
|
157
|
+
out_dict['geos_versions'] = geos_versions
|
|
158
|
+
return out_dict
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _read_mod_file_geos_sources(mod_file: str) -> Dict[str, GeosVersion]:
|
|
162
|
+
nhead = mod_utils.get_num_header_lines(mod_file)
|
|
163
|
+
sources = dict()
|
|
164
|
+
with open(mod_file) as f:
|
|
165
|
+
for idx, line in enumerate(f):
|
|
166
|
+
if idx == nhead:
|
|
167
|
+
return sources
|
|
168
|
+
elif line.startswith('GEOS source'):
|
|
169
|
+
# Assume a line like "GEOS source : Met3d : {version info} : {filename} : {checksum}"
|
|
170
|
+
parts = line.split(':', maxsplit=2)
|
|
171
|
+
key = parts[1].strip()
|
|
172
|
+
info = parts[2].strip()
|
|
173
|
+
sources[key] = GeosVersion.from_str(info)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _read_mod_file_co_source(mod_file: str, geos_versions: Dict[str, GeosVersion]) -> GeosSource:
|
|
178
|
+
# >= v1.2.1 .mod files (or those patched to look like them) have all GEOS sources in the header,
|
|
179
|
+
# so we don't need to re-read the file.
|
|
180
|
+
if geos_versions:
|
|
181
|
+
chm_version = geos_versions.get('Chm3d')
|
|
182
|
+
return GeosSource.UNKNOWN if chm_version is None else chm_version.source
|
|
183
|
+
|
|
184
|
+
# v1.2.0 files have the line "CO source" in the header and pre-v1.2.0 files have no indication
|
|
185
|
+
# of version.
|
|
186
|
+
nhead = mod_utils.get_num_header_lines(mod_file)
|
|
187
|
+
with open(mod_file) as f:
|
|
188
|
+
for idx, line in enumerate(f):
|
|
189
|
+
if idx == nhead:
|
|
190
|
+
# This is the default; .mod files before v1.2.0 did not include a CO
|
|
191
|
+
# source in the header because it was always from GEOS FP-IT. Also
|
|
192
|
+
# prior to v1.2.0, the header only had 7 lines (by default), so if
|
|
193
|
+
# there's >7 lines, that probably means we messed up.
|
|
194
|
+
if nhead > 7:
|
|
195
|
+
logger.warning((f'In .mod file {mod_file}, did not find a "CO source" line in the header, but the header has more than 7 lines. '
|
|
196
|
+
'Unless this is a custom .mod file, this means I may have missed the CO source line.'))
|
|
197
|
+
return GeosSource.FPIT
|
|
198
|
+
elif line.startswith('CO source'):
|
|
199
|
+
source = line.split(':', maxsplit=2)[1].strip()
|
|
200
|
+
return GeosSource(source)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def read_mod_file_units(mod_file):
|
|
204
|
+
"""
|
|
205
|
+
Get the units for the profile variables in a .mod file
|
|
206
|
+
|
|
207
|
+
:param mod_file: the .mod file to read
|
|
208
|
+
:type mod_file: str
|
|
209
|
+
|
|
210
|
+
:return: a dictionary with the variable names as keys and the units as values.
|
|
211
|
+
"""
|
|
212
|
+
n_header_lines = mod_utils.get_num_header_lines(mod_file)
|
|
213
|
+
# Assume that the profile units are the second to last line of the header
|
|
214
|
+
# and the profile variable names are the last line
|
|
215
|
+
with open(mod_file, 'r') as robj:
|
|
216
|
+
for i in range(n_header_lines):
|
|
217
|
+
line = robj.readline()
|
|
218
|
+
if i == (n_header_lines-2):
|
|
219
|
+
units = line.split()
|
|
220
|
+
elif i == (n_header_lines-1):
|
|
221
|
+
names = line.split()
|
|
222
|
+
|
|
223
|
+
return {n: u for n, u in zip(names, units)}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def read_map_file(map_file, as_dataframes=False, skip_header=False, fmt=None):
|
|
227
|
+
"""
|
|
228
|
+
Read a .map file
|
|
229
|
+
|
|
230
|
+
:param map_file: the path to the .map file
|
|
231
|
+
:type map_file: str
|
|
232
|
+
|
|
233
|
+
:param as_dataframes: set to ``True`` to return the constants and profiles data as Pandas dataframes. By default,
|
|
234
|
+
(``False``) they are returned as dictionaries of numpy arrays.
|
|
235
|
+
:type as_dataframes: bool
|
|
236
|
+
|
|
237
|
+
:param skip_header: set to ``True` to avoid reading the header. This is helpful for reading .map files that
|
|
238
|
+
have a slightly different header format.
|
|
239
|
+
:type skip_header: bool
|
|
240
|
+
|
|
241
|
+
:param fmt: can use to tell this function which file type the .map file is, either "txt" for text or "nc" for
|
|
242
|
+
netCDF. If not specified, it will attempt to infer from the extension.
|
|
243
|
+
:type fmt: str or None
|
|
244
|
+
|
|
245
|
+
:return: a dictionary with keys 'constants' and 'profile' that hold the header values and main profile data,
|
|
246
|
+
respectively. The form of these values depends on ``as_dataframes``.
|
|
247
|
+
:rtype: dict
|
|
248
|
+
"""
|
|
249
|
+
if map_file.endswith('.nc') or fmt == 'nc':
|
|
250
|
+
return _read_map_nc_file(map_file, as_dataframes=as_dataframes, skip_header=skip_header)
|
|
251
|
+
elif map_file.endswith('.map') or fmt == 'txt':
|
|
252
|
+
return _read_map_txt_file(map_file, as_dataframes=as_dataframes, skip_header=skip_header)
|
|
253
|
+
else:
|
|
254
|
+
ext = os.path.splitext(map_file)
|
|
255
|
+
raise ValueError('Unknown extension "{}" for map file. Use the `fmt` keyword to indicate the file type.'.format(ext))
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _read_map_nc_file(map_file, as_dataframes=False, skip_header=False):
|
|
259
|
+
profile_dict = dict()
|
|
260
|
+
constant_dict = dict()
|
|
261
|
+
with ncdf.Dataset(map_file) as ds:
|
|
262
|
+
for varname, vardat in ds.variables.items():
|
|
263
|
+
if varname == 'time':
|
|
264
|
+
vardat = ncdf.num2date(vardat[:], vardat.units, only_use_cftime_datetimes=False)
|
|
265
|
+
profile_dict[varname] = pd.DatetimeIndex(vardat)
|
|
266
|
+
else:
|
|
267
|
+
profile_dict[varname] = vardat[:].filled(np.nan)
|
|
268
|
+
|
|
269
|
+
if not skip_header:
|
|
270
|
+
for attr in ds.ncattrs():
|
|
271
|
+
if attr.startswith('constant') and not attr.endswith('units'):
|
|
272
|
+
k = attr.replace('constant_', '')
|
|
273
|
+
constant_dict[k] = ds.getncattr(attr)
|
|
274
|
+
|
|
275
|
+
if as_dataframes:
|
|
276
|
+
# these are both scalar values that can't go into the dataframe
|
|
277
|
+
profile_dict.pop('time')
|
|
278
|
+
profile_dict.pop('lat')
|
|
279
|
+
profile_dict = pd.DataFrame(profile_dict)
|
|
280
|
+
profile_dict.set_index('altitude', inplace=True)
|
|
281
|
+
|
|
282
|
+
constant_dict = pd.DataFrame(constant_dict, index=[0])
|
|
283
|
+
|
|
284
|
+
return {'profile': profile_dict, 'constants': constant_dict}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _read_map_txt_file(map_file, as_dataframes=False, skip_header=False):
|
|
288
|
+
n_header_lines = mod_utils.get_num_header_lines(map_file)
|
|
289
|
+
constants = dict()
|
|
290
|
+
if not skip_header:
|
|
291
|
+
with open(map_file, 'r') as mapf:
|
|
292
|
+
n_skip = 4
|
|
293
|
+
# Skip the first four lines to get to the constants - these should be (1) the number of header lines &
|
|
294
|
+
# columns, (2) filename, (3) version info, and (4) wiki reference.
|
|
295
|
+
for i in range(n_skip):
|
|
296
|
+
mapf.readline()
|
|
297
|
+
|
|
298
|
+
# The last two lines of the header are the column names and units; everything between line 5 and that should
|
|
299
|
+
# be physical constants. Start at n_skip+1 to account for 0 indexing vs. number of lines.
|
|
300
|
+
|
|
301
|
+
for i in range(n_skip+1, n_header_lines-1):
|
|
302
|
+
line = mapf.readline()
|
|
303
|
+
# Lines have the form Name (units): value - ignore anything in parentheses
|
|
304
|
+
name, value = line.split(':')
|
|
305
|
+
name = re.sub(r'\(.+\)', '', name).strip()
|
|
306
|
+
constants[name] = float(value)
|
|
307
|
+
|
|
308
|
+
df = pd.read_csv(map_file, header=n_header_lines-2, skiprows=[n_header_lines-1], na_values='NAN')
|
|
309
|
+
# Sometimes extra space gets kept in the headers - remove that
|
|
310
|
+
df.rename(columns=lambda h: h.strip(), inplace=True)
|
|
311
|
+
if not as_dataframes:
|
|
312
|
+
data = {k: v.values for k, v in df.items()}
|
|
313
|
+
else:
|
|
314
|
+
data = df
|
|
315
|
+
constants = pd.DataFrame(constants, index=[0])
|
|
316
|
+
|
|
317
|
+
out_dict = dict()
|
|
318
|
+
out_dict['constants'] = constants
|
|
319
|
+
out_dict['profile'] = data
|
|
320
|
+
return out_dict
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def read_isotopes(isotopes_file, gases_only=False):
|
|
324
|
+
"""
|
|
325
|
+
Read the isotopes defined in an isotopologs.dat file
|
|
326
|
+
|
|
327
|
+
:param isotopes_file: the path to the isotopologs.dat file
|
|
328
|
+
:type isotopes_file: str
|
|
329
|
+
|
|
330
|
+
:param gases_only: set to ``True`` to return a tuple of only the distinct gases, not the individual isotopes.
|
|
331
|
+
Default is ``False``, which includes the different isotope numbers.
|
|
332
|
+
:type gases_only: bool
|
|
333
|
+
|
|
334
|
+
:return: tuple of isotope or gas names
|
|
335
|
+
:rtype: tuple(str)
|
|
336
|
+
"""
|
|
337
|
+
nheader = mod_utils.get_num_header_lines(isotopes_file)
|
|
338
|
+
with open(isotopes_file, 'r') as fobj:
|
|
339
|
+
for i in range(nheader):
|
|
340
|
+
fobj.readline()
|
|
341
|
+
|
|
342
|
+
isotopes = []
|
|
343
|
+
for line in fobj:
|
|
344
|
+
iso_number = line[3:5].strip()
|
|
345
|
+
iso_name = line[6:14].strip()
|
|
346
|
+
if not gases_only:
|
|
347
|
+
iso_name = iso_number + iso_name
|
|
348
|
+
if iso_name not in isotopes:
|
|
349
|
+
isotopes.append(iso_name)
|
|
350
|
+
|
|
351
|
+
return tuple(isotopes)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def read_vmr_file(vmr_file, as_dataframes=False, lowercase_names=True, style='new'):
|
|
355
|
+
nheader = mod_utils.get_num_header_lines(vmr_file)
|
|
356
|
+
|
|
357
|
+
if style == 'new':
|
|
358
|
+
last_const_line = nheader - 1
|
|
359
|
+
old_style = False
|
|
360
|
+
elif style == 'old':
|
|
361
|
+
last_const_line = 4
|
|
362
|
+
old_style = True
|
|
363
|
+
else:
|
|
364
|
+
raise ValueError('style must be one of "new" or "old"')
|
|
365
|
+
|
|
366
|
+
header_data = dict()
|
|
367
|
+
with open(vmr_file, 'r') as fobj:
|
|
368
|
+
# Skip the line with the number of header lines and columns
|
|
369
|
+
fobj.readline()
|
|
370
|
+
for i in range(1, last_const_line):
|
|
371
|
+
line = fobj.readline()
|
|
372
|
+
const_name, const_val = [v.strip() for v in line.split(':')]
|
|
373
|
+
if lowercase_names:
|
|
374
|
+
const_name = const_name.lower()
|
|
375
|
+
|
|
376
|
+
try:
|
|
377
|
+
const_val = float(const_val)
|
|
378
|
+
except ValueError:
|
|
379
|
+
pass
|
|
380
|
+
header_data[const_name] = const_val
|
|
381
|
+
|
|
382
|
+
prior_info = dict()
|
|
383
|
+
if old_style:
|
|
384
|
+
for i in range(last_const_line, nheader-1, 2):
|
|
385
|
+
category_line = fobj.readline()
|
|
386
|
+
category = re.split(r'[:\.]', category_line)[0].strip()
|
|
387
|
+
data_line = fobj.readline()
|
|
388
|
+
data_line = data_line.split(':')[1].strip()
|
|
389
|
+
split_data_line = re.split(r'\s+', data_line)
|
|
390
|
+
prior_info[category] = np.array([float(x) for x in split_data_line])
|
|
391
|
+
|
|
392
|
+
data_table = pd.read_csv(vmr_file, sep='\s+', header=nheader-1)
|
|
393
|
+
|
|
394
|
+
# Also get the information that's only in the file name (namely date and longitude, we'll also read the latitude
|
|
395
|
+
# because it's there).
|
|
396
|
+
file_vars = dict()
|
|
397
|
+
base_name = os.path.basename(vmr_file)
|
|
398
|
+
try:
|
|
399
|
+
file_vars['datetime'] = mod_utils.find_datetime_substring(base_name, out_type=dt.datetime)
|
|
400
|
+
file_vars['lon'] = mod_utils.find_lon_substring(base_name, to_float=True)
|
|
401
|
+
file_vars['lat'] = mod_utils.find_lat_substring(base_name, to_float=True)
|
|
402
|
+
except AttributeError:
|
|
403
|
+
# Happens when the regex can't find a date/lon/lat in the file name
|
|
404
|
+
# usually means we're reading a climatological file
|
|
405
|
+
file_vars = dict(datetime=None, lon=None, lat=None)
|
|
406
|
+
|
|
407
|
+
if lowercase_names:
|
|
408
|
+
data_table.columns = [v.lower() for v in data_table]
|
|
409
|
+
|
|
410
|
+
if as_dataframes:
|
|
411
|
+
header_data = pd.DataFrame(header_data, index=[0])
|
|
412
|
+
# Rearrange the prior info dict so that the data frame has the categories as the index and the species as the
|
|
413
|
+
# columns.
|
|
414
|
+
categories = list(prior_info.keys())
|
|
415
|
+
tmp_prior_info = dict()
|
|
416
|
+
for i, k in enumerate(data_table.columns.drop('altitude')):
|
|
417
|
+
tmp_prior_info[k] = np.array([prior_info[cat][i] for cat in categories])
|
|
418
|
+
prior_info = pd.DataFrame(tmp_prior_info, index=categories)
|
|
419
|
+
else:
|
|
420
|
+
# use an ordered dict to ensure we keep the order of the gases. This is important if we use this .vmr file as
|
|
421
|
+
# a template to write another .vmr file that gsetup.f can read.
|
|
422
|
+
data_table = OrderedDict([(k, v.to_numpy()) for k, v in data_table.items()])
|
|
423
|
+
|
|
424
|
+
return {'scalar': header_data, 'profile': data_table, 'prior_info': prior_info, 'file': file_vars}
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def read_runlog(runlog_file, as_dataframes=False, remove_commented_lines=True):
|
|
428
|
+
nhead = mod_utils.get_num_header_lines(runlog_file)
|
|
429
|
+
with open(runlog_file, 'r') as robj:
|
|
430
|
+
for i in range(nhead-2):
|
|
431
|
+
# Skip ahead to the last two lines of the header which contain the fortran format specification and the
|
|
432
|
+
# column names
|
|
433
|
+
robj.readline()
|
|
434
|
+
|
|
435
|
+
fmt_line = robj.readline()
|
|
436
|
+
# we need to read the column names twice. read_fwf expects the first line it reads to be the column names but
|
|
437
|
+
# with the right widths. Since that's not the case for the runlog, we read in the line manually once (to parse
|
|
438
|
+
# later into the proper column names) then rewind the file pointed to the beginning of that line so that
|
|
439
|
+
# read_fwf reads it in as the column names. It'll get them wrong, but at least it doesn't stick a data row in
|
|
440
|
+
# as the header.
|
|
441
|
+
pos = robj.tell()
|
|
442
|
+
column_names_line = robj.readline()
|
|
443
|
+
robj.seek(pos)
|
|
444
|
+
|
|
445
|
+
# the format line will be something like: "format=(a1,a57,1x,2i4,f8.4,f8.3,f9.3,...)\n"; we want to pass only
|
|
446
|
+
# the part inside the parentheses (including the parentheses)
|
|
447
|
+
colspecs, _ = mod_utils.fortran_fmt_to_fwf_tuples(fmt_line.split('=')[1].strip())
|
|
448
|
+
df = pd.read_fwf(robj, colspecs=colspecs)
|
|
449
|
+
|
|
450
|
+
# Fix the header. The first column is just where the colon goes if the line is commented out so we have to give
|
|
451
|
+
# that a name (there isn't one in the header)
|
|
452
|
+
column_names = column_names_line.split()
|
|
453
|
+
df.columns = ['comment'] + column_names_line.split()
|
|
454
|
+
|
|
455
|
+
# Remove comments if requested.
|
|
456
|
+
if remove_commented_lines:
|
|
457
|
+
df = df[~(df['comment'].apply(lambda x: x == ':'))]
|
|
458
|
+
|
|
459
|
+
# Always remove the comment column itself
|
|
460
|
+
df = df[column_names]
|
|
461
|
+
|
|
462
|
+
if as_dataframes:
|
|
463
|
+
return df
|
|
464
|
+
else:
|
|
465
|
+
return df.to_dict()
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def read_tabular_file_with_header(file_path: Union[str, Path], comment_str: str = '#') -> pd.DataFrame:
|
|
469
|
+
with open(file_path) as f:
|
|
470
|
+
for line in f:
|
|
471
|
+
if not line.startswith(comment_str):
|
|
472
|
+
break
|
|
473
|
+
|
|
474
|
+
columns = line.strip().split()
|
|
475
|
+
df = pd.read_csv(f, header=None, delim_whitespace=True)
|
|
476
|
+
df.columns = columns
|
|
477
|
+
return df
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from . import mod_utils, readers
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def iter_runlog_args(runlog, first_date='2000-01-01', site_abbrv=None):
|
|
9
|
+
def _reduce_lat_lons(df):
|
|
10
|
+
slla = []
|
|
11
|
+
for (site, lon, lat), ll_df in df.groupby(['site', 'oblon', 'oblat']):
|
|
12
|
+
if not np.isclose(df['obalt'], df['obalt'][0]).all():
|
|
13
|
+
lon_str = mod_utils.format_lon(lon)
|
|
14
|
+
lat_str = mod_utils.format_lat(lat)
|
|
15
|
+
first_date = ll_df.index.min().strftime('%Y-%m-%d %H:%M:%S')
|
|
16
|
+
last_date = ll_df.index.min().strftime('%Y-%m-%d %H:%M:%S')
|
|
17
|
+
print('WARNING: multiple observation altitudes for {lon}, {lat} in the date range {start} to {stop}. '
|
|
18
|
+
'Will only produce the .mod files for the lowest altitude. (Note: altitude only matters if '
|
|
19
|
+
'generating .mod files for a slant path.)'.format(lon=lon_str, lat=lat_str, start=first_date,
|
|
20
|
+
stop=last_date),
|
|
21
|
+
file=sys.stderr)
|
|
22
|
+
|
|
23
|
+
slla.append((site, lon, lat, df['obalt'].min()))
|
|
24
|
+
# convert to lists of sites, lons, lats, and alts separately
|
|
25
|
+
return zip(*slla)
|
|
26
|
+
|
|
27
|
+
rldf = readers.read_runlog(runlog, as_dataframes=True)
|
|
28
|
+
rl_dates = mod_utils.ydh_to_date(rldf['Year'], rldf['Day'], rldf['Hour'])
|
|
29
|
+
rldf.set_index(rl_dates, inplace=True)
|
|
30
|
+
|
|
31
|
+
# GEOS FPIT has no data before 2000 so we usually remove those dates
|
|
32
|
+
if first_date:
|
|
33
|
+
xx_dates = rl_dates >= first_date
|
|
34
|
+
rldf = rldf[xx_dates]
|
|
35
|
+
|
|
36
|
+
date_ranges = mod_utils.get_runlog_geos_date_ranges(rldf)
|
|
37
|
+
|
|
38
|
+
if site_abbrv is None:
|
|
39
|
+
# If no site abbreviation given, assume that the site abbreviations are the first two letters of each spectrum
|
|
40
|
+
site_abbrv = [s[:2] for s in rldf['Spectrum_File_Name']]
|
|
41
|
+
elif isinstance(site_abbrv, str):
|
|
42
|
+
# If a single site abbreviation given, then it must be used for all sites, so we need to expand it so that we
|
|
43
|
+
# can create a date-indexed series to subset as we loop through the date ranges
|
|
44
|
+
site_abbrv = rldf.shape[0] * [site_abbrv]
|
|
45
|
+
|
|
46
|
+
rldf['site'] = site_abbrv
|
|
47
|
+
|
|
48
|
+
for drange in date_ranges:
|
|
49
|
+
dstart = (drange[0] - pd.Timedelta(minutes=90)).strftime('%Y-%m-%d %H:%M:%S')
|
|
50
|
+
dstop = drange[1].strftime('%Y-%m-%d %H:%M:%S') # end time is already one GEOS time step ahead
|
|
51
|
+
xx = slice(dstart, dstop)
|
|
52
|
+
sub_df = rldf.loc[xx, ['oblon', 'oblat', 'obalt', 'site']]
|
|
53
|
+
if sub_df.shape[0] == 0:
|
|
54
|
+
raise RuntimeError('Found no runlog entries between {} and {}; error in calculated date ranges?'
|
|
55
|
+
.format(dstart, dstop))
|
|
56
|
+
abbrv, lon, lat, alt = _reduce_lat_lons(sub_df)
|
|
57
|
+
yield drange, abbrv, lon, lat, alt
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def time_weight_from_datetime(acos_datetime, prev_geos_datetime, next_geos_datetime):
|
|
5
|
+
"""
|
|
6
|
+
Calculate the time weighting from datetime-like objects.
|
|
7
|
+
|
|
8
|
+
:param acos_datetime: the date/time of the ACOS sounding
|
|
9
|
+
:type acos_datetime: datetime-like
|
|
10
|
+
|
|
11
|
+
:param prev_geos_datetime: the date/time of the GEOS FP file that comes before the ACOS sounding
|
|
12
|
+
:type prev_geos_datetime: datetime-like
|
|
13
|
+
|
|
14
|
+
:param next_geos_datetime: the date/time of the GEOS FP file that comes after the ACOS sounding
|
|
15
|
+
:type next_geos_datetime: datetime-like
|
|
16
|
+
|
|
17
|
+
:return: the weight, w, such that the profile for the ACOS time would be :math:`w*p1 + (1-w)*p2`, where p1 and p2
|
|
18
|
+
are the profiles at the GEOS times before and after the sounding, respectively.
|
|
19
|
+
:rtype: float
|
|
20
|
+
"""
|
|
21
|
+
return time_weight(datetime2datenum(acos_datetime), datetime2datenum(prev_geos_datetime), datetime2datenum(next_geos_datetime))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def time_weight(acos_datenum, prev_geos_datenum, next_geos_datenum):
|
|
25
|
+
"""
|
|
26
|
+
Calculate the time weighting from date numbers.
|
|
27
|
+
|
|
28
|
+
In this context, a date number is any representation of a date time as a integer- or float-like value. Typically,
|
|
29
|
+
this will be a value returned by :func:`datetime2datenum` but any similar numeric value will do, so long as all
|
|
30
|
+
inputs are consistent.
|
|
31
|
+
|
|
32
|
+
:param acos_datenum: the date/time of the ACOS sounding as a number
|
|
33
|
+
:type acos_datenum: int or float
|
|
34
|
+
|
|
35
|
+
:param prev_geos_datenum: the date/time of the GEOS FP file that comes before the ACOS sounding as a number
|
|
36
|
+
:type prev_geos_datenum: int or float
|
|
37
|
+
|
|
38
|
+
:param next_geos_datenum: the date/time of the GEOS FP file that comes after the ACOS sounding as a number
|
|
39
|
+
:type next_geos_datenum: int or float
|
|
40
|
+
|
|
41
|
+
:return: the weight, w, such that the profile for the ACOS time would be :math:`w*p1 + (1-w)*p2`, where p1 and p2
|
|
42
|
+
are the profiles at the GEOS times before and after the sounding, respectively.
|
|
43
|
+
:rtype: float
|
|
44
|
+
"""
|
|
45
|
+
# if acos_datenum == prev_geos_datenum, then this == 1 => p = 1*p1 + 0*p2 = p1
|
|
46
|
+
# if acos_datenum == next_geos_datenum, then this == 0 => p = 0*p1 + 1*p2 = p2
|
|
47
|
+
return (next_geos_datenum - acos_datenum) / (next_geos_datenum - prev_geos_datenum)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def datetime2datenum(datetime_obj):
|
|
51
|
+
"""
|
|
52
|
+
Convert a single :class:`datetime.datetime` object into a date number.
|
|
53
|
+
|
|
54
|
+
Internally, this converts the datetime object into a :class:`numpy.datetime64` object with units of seconds, then
|
|
55
|
+
converts that to a float type. Under numpy version 1.16.2, this results in a number that is seconds since 1 Jan
|
|
56
|
+
1970; however, I have not seen any documentation from numpy guaranteeing that behavior. Therefore, any use of these
|
|
57
|
+
date numbers should be careful to verify this behavior. The following assert block can be used to check this::
|
|
58
|
+
|
|
59
|
+
assert numpy.isclose(datetime2datenum('1970-01-01'), 0.0) and numpy.isclose(datetime2datenum('1970-01-02'), 86400)
|
|
60
|
+
|
|
61
|
+
:param datetime_obj: the datetime to convert. May be any type that :class:`numpy.datetime64` can intepret as a
|
|
62
|
+
datetime.
|
|
63
|
+
|
|
64
|
+
:return: the converted date number
|
|
65
|
+
:rtype: :class:`numpy.float`
|
|
66
|
+
"""
|
|
67
|
+
return np.datetime64(datetime_obj, 's').astype(float)
|