ginput 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. ginput/__init__.py +4 -0
  2. ginput/common_utils/__init__.py +0 -0
  3. ginput/common_utils/ggg_logging.py +62 -0
  4. ginput/common_utils/ioutils.py +304 -0
  5. ginput/common_utils/mod_constants.py +24 -0
  6. ginput/common_utils/mod_utils.py +2663 -0
  7. ginput/common_utils/readers.py +477 -0
  8. ginput/common_utils/run_utils.py +57 -0
  9. ginput/common_utils/sat_utils.py +67 -0
  10. ginput/common_utils/versioning.py +251 -0
  11. ginput/common_utils/writers.py +377 -0
  12. ginput/data/.gitignore +1 -0
  13. ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
  14. ginput/data/ML_monthly_obs_ch4.txt +498 -0
  15. ginput/data/ML_monthly_obs_co.txt +412 -0
  16. ginput/data/ML_monthly_obs_co2.txt +597 -0
  17. ginput/data/ML_monthly_obs_n2o.txt +330 -0
  18. ginput/data/SMO_monthly_obs_ch4.txt +499 -0
  19. ginput/data/SMO_monthly_obs_co.txt +416 -0
  20. ginput/data/SMO_monthly_obs_co2.txt +613 -0
  21. ginput/data/SMO_monthly_obs_n2o.txt +330 -0
  22. ginput/data/ace_fn2o_lut.nc +0 -0
  23. ginput/data/age_spectra/age.midlat.txt +47 -0
  24. ginput/data/age_spectra/age.tropics.txt +46 -0
  25. ginput/data/age_spectra/age.vortex.txt +31 -0
  26. ginput/data/age_spectra/agespec.midlat.txt +47 -0
  27. ginput/data/age_spectra/agespec.tropics.txt +46 -0
  28. ginput/data/age_spectra/agespec.vortex.txt +31 -0
  29. ginput/data/age_spectra/time.txt +3001 -0
  30. ginput/data/ch4_hf_slopes.nc +0 -0
  31. ginput/data/ch4_strat_lut.nc +0 -0
  32. ginput/data/clams_age_clim_scaled.nc +0 -0
  33. ginput/data/co2_strat_lut.nc +0 -0
  34. ginput/data/fo2_data/.gitignore +2 -0
  35. ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
  36. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
  37. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
  38. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
  39. ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
  40. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
  41. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
  42. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
  43. ginput/data/hf_strat_lut.nc +0 -0
  44. ginput/data/isotopologs.dat +199 -0
  45. ginput/data/meso_co_lut.nc +0 -0
  46. ginput/data/n2o_ch4_acefts.nc +0 -0
  47. ginput/data/n2o_strat_lut.nc +0 -0
  48. ginput/data/o2_mean_dmf.dat +52 -0
  49. ginput/data/summer_35N.vmr +182 -0
  50. ginput/data/washenfelder03_table_s3.txt +83 -0
  51. ginput/download/__init__.py +0 -0
  52. ginput/download/download_utils.py +46 -0
  53. ginput/download/get_GEOS5.py +367 -0
  54. ginput/download/get_MERRA2.py +177 -0
  55. ginput/download/get_NOAA_flask_data.py +103 -0
  56. ginput/download/get_fo2_data.py +140 -0
  57. ginput/mod_maker/__init__.py +0 -0
  58. ginput/mod_maker/mod_maker.py +2470 -0
  59. ginput/mod_maker/slantify.py +450 -0
  60. ginput/mod_maker/tccon_sites.py +570 -0
  61. ginput/priors/__init__.py +0 -0
  62. ginput/priors/acos_interface.py +1376 -0
  63. ginput/priors/automation.py +313 -0
  64. ginput/priors/backend_analysis/__init__.py +0 -0
  65. ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
  66. ginput/priors/backend_analysis/ace_fts_val.py +452 -0
  67. ginput/priors/backend_analysis/acos_comparison.py +550 -0
  68. ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
  69. ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
  70. ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
  71. ginput/priors/backend_analysis/backend_driver.py +90 -0
  72. ginput/priors/backend_analysis/backend_utils.py +358 -0
  73. ginput/priors/backend_analysis/clams.py +71 -0
  74. ginput/priors/backend_analysis/create_test_priors.py +853 -0
  75. ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
  76. ginput/priors/backend_analysis/mls_val.py +217 -0
  77. ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
  78. ginput/priors/fo2_prep.py +311 -0
  79. ginput/priors/map_maker.py +193 -0
  80. ginput/priors/mlo_smo_prep.py +1146 -0
  81. ginput/priors/tccon_priors.py +3482 -0
  82. ginput-1.3.0.dist-info/LICENSE.txt +203 -0
  83. ginput-1.3.0.dist-info/METADATA +125 -0
  84. ginput-1.3.0.dist-info/RECORD +86 -0
  85. ginput-1.3.0.dist-info/WHEEL +5 -0
  86. ginput-1.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,477 @@
1
+ from collections import OrderedDict
2
+ import datetime as dt
3
+ import os
4
+ from pathlib import Path
5
+ import re
6
+
7
+ import netCDF4 as ncdf
8
+ import numpy as np
9
+ import pandas as pd
10
+
11
+ from . import mod_utils
12
+ from .mod_utils import ModelError
13
+ from .versioning import GeosVersion, GeosSource
14
+ from .ggg_logging import logger
15
+
16
+ from typing import Dict, Union
17
+
18
+ def read_out_file(out_file, as_dataframes=False, replace_fills=False):
19
+ """Read a GGG output file
20
+
21
+ This function can read most GGG output files, including .col files and post processing
22
+ text files (but not the .private.nc file). It expects a file where:
23
+
24
+ * the first line is a sequence of numbers, with the first one being the number of header
25
+ lines
26
+ * the last header line is a space-separated list of column names
27
+ * the rest of the file is a space-separated data table.
28
+
29
+ Parameters
30
+ ----------
31
+ out_file : str
32
+ Path to the file to read
33
+
34
+ as_dataframes : bool
35
+ If `True`, then the output file is read in as a dataframe. If `False`, it is converted
36
+ to a dictionary when returning.
37
+
38
+ replace_fills : bool or float
39
+ If `False`, fill values are kept as is. If `True`, this will look for a line in the header
40
+ with the format `"missing: <number>`. Any number within floating point rounding of that
41
+ number are replaced with NaNs. If this is a number, then any values greater than or equal
42
+ to that number are treated as fills and replaced with NaNs. This last is the safest; GGG
43
+ post processing typically uses numbers >1e35 as "missing" values, so usually any value greater
44
+ than that can be replaced.
45
+
46
+ Note that `TypeError` occurences during comparison are ignored. This allows e.g. the "spectrum"
47
+ column to be skipped, but means that any formatting issues that cause a column to be read in
48
+ as an series of objects, rather than a numeric type, can cause that column to be skipped.
49
+
50
+ Returns
51
+ -------
52
+ pandas.DataFrame or dict
53
+ The contents of the file as a dataframe or dict, depending on `as_dataframes`.
54
+ """
55
+ n_header_lines = mod_utils.get_num_header_lines(out_file)
56
+ df = pd.read_csv(out_file, header=n_header_lines-1, sep='\s+')
57
+
58
+ if replace_fills is True:
59
+ # Get the fill value from the header
60
+ with open(out_file) as f:
61
+ fill_value = None
62
+ for _ in range(n_header_lines):
63
+ line = f.readline()
64
+ if line.startswith('missing:'):
65
+ fill_value = float(line.split(':', 1)[1])
66
+ break
67
+
68
+ if fill_value is None:
69
+ raise IOError('Could not find fill value in the header of {}'.format(out_file))
70
+
71
+ is_fill = lambda val: np.isclose(val, fill_value)
72
+
73
+ elif replace_fills is not False:
74
+ fill_value = replace_fills
75
+ replace_fills = True
76
+ is_fill = lambda val: val >= fill_value
77
+
78
+ if replace_fills:
79
+ for colname, coldata in df.iteritems():
80
+ try:
81
+ xx_fills = is_fill(coldata)
82
+ except TypeError:
83
+ continue
84
+ else:
85
+ df.loc[xx_fills, colname] = np.nan
86
+
87
+ if not as_dataframes:
88
+ return df.to_dict()
89
+ else:
90
+ return df
91
+
92
+
93
+ def read_mod_file(mod_file, as_dataframes=False):
94
+ """
95
+ Read a TCCON .mod file.
96
+
97
+ :param mod_file: the path to the mod file.
98
+ :type mod_file: str
99
+
100
+ :param as_dataframes: if ``True``, then the collection of variables will be kept as dataframes. If ``False``
101
+ (default), they are converted to dictionaries of floats or numpy arrays.
102
+ :type as_dataframes: bool
103
+
104
+ :return: a dictionary with keys 'file' (values derived from file name), 'constants' (constant values stored in the
105
+ .mod file header), 'scalar' (values like surface height and tropopause pressure that are only defined once per
106
+ profile) and 'profile' (profile variables) containing the respective variables. These values will be dictionaries
107
+ or data frames, depending on ``as_dataframes``.
108
+ :rtype: dict
109
+ """
110
+ n_header_lines = mod_utils.get_num_header_lines(mod_file)
111
+ # Read the constants from the second line of the file. There's no header for these, we just have to rely on the
112
+ # same constants being in the same position.
113
+ constant_vars = pd.read_csv(mod_file, sep=r'\s+', header=None, nrows=1, skiprows=1,
114
+ names=('earth_radius', 'ecc2', 'obs_lat', 'surface_gravity',
115
+ 'profile_base_geometric_alt', 'base_pressure', 'tropopause_pressure'))
116
+ # Read the scalar variables (e.g. surface pressure, SZA, tropopause) first. We just have to assume their headers are
117
+ # on line 3 and values on line 4 of the file, the first number in the first line gives us the line the profile
118
+ # variables start on.
119
+ scalar_vars = pd.read_csv(mod_file, sep=r'\s+', header=2, nrows=1)
120
+
121
+ # Get the GEOS versions from the header. Also extract the CO source, since the priors
122
+ # code relies on that and we should support pre-version 1.2.1 files.
123
+ geos_versions = _read_mod_file_geos_sources(mod_file)
124
+ constant_vars['co_source'] = _read_mod_file_co_source(mod_file, geos_versions)
125
+
126
+ # Now read the profile vars.
127
+ profile_vars = pd.read_csv(mod_file, sep=r'\s+', header=n_header_lines-1)
128
+
129
+ # Also get the information that's only in the file name (namely date and longitude, we'll also read the latitude
130
+ # because it's there).
131
+ file_vars = dict()
132
+ base_name = os.path.basename(mod_file)
133
+ file_vars['datetime'] = mod_utils.find_datetime_substring(base_name, out_type=dt.datetime)
134
+ file_vars['lon'] = mod_utils.find_lon_substring(base_name, to_float=True)
135
+ file_vars['lat'] = mod_utils.find_lat_substring(base_name, to_float=True)
136
+
137
+ # Check that the header latitude and the file name latitude don't differ by more than 0.5 degree. Even if rounded
138
+ # to an integer for the file name, the difference should not exceed 0.5 degree.
139
+ lat_diff_threshold = 0.5
140
+ if np.abs(file_vars['lat'] - constant_vars['obs_lat'].item()) > lat_diff_threshold:
141
+ raise ModelError('The latitude in the file name and .mod file header differ by more than {lim} deg ({name} vs. '
142
+ '{head}). This indicates a possibly malformed .mod file.'
143
+ .format(lim=lat_diff_threshold, name=file_vars['lat'], head=constant_vars['obs_lat'].item())
144
+ )
145
+
146
+ out_dict = dict()
147
+ if as_dataframes:
148
+ out_dict['file'] = pd.DataFrame(file_vars, index=[0])
149
+ out_dict['constants'] = constant_vars
150
+ out_dict['scalar'] = scalar_vars
151
+ out_dict['profile'] = profile_vars
152
+ else:
153
+ out_dict['file'] = file_vars
154
+ out_dict['constants'] = {k: v.item() for k, v in constant_vars.items()}
155
+ out_dict['scalar'] = {k: v.item() for k, v in scalar_vars.items()}
156
+ out_dict['profile'] = {k: v.values for k, v in profile_vars.items()}
157
+ out_dict['geos_versions'] = geos_versions
158
+ return out_dict
159
+
160
+
161
+ def _read_mod_file_geos_sources(mod_file: str) -> Dict[str, GeosVersion]:
162
+ nhead = mod_utils.get_num_header_lines(mod_file)
163
+ sources = dict()
164
+ with open(mod_file) as f:
165
+ for idx, line in enumerate(f):
166
+ if idx == nhead:
167
+ return sources
168
+ elif line.startswith('GEOS source'):
169
+ # Assume a line like "GEOS source : Met3d : {version info} : {filename} : {checksum}"
170
+ parts = line.split(':', maxsplit=2)
171
+ key = parts[1].strip()
172
+ info = parts[2].strip()
173
+ sources[key] = GeosVersion.from_str(info)
174
+
175
+
176
+
177
+ def _read_mod_file_co_source(mod_file: str, geos_versions: Dict[str, GeosVersion]) -> GeosSource:
178
+ # >= v1.2.1 .mod files (or those patched to look like them) have all GEOS sources in the header,
179
+ # so we don't need to re-read the file.
180
+ if geos_versions:
181
+ chm_version = geos_versions.get('Chm3d')
182
+ return GeosSource.UNKNOWN if chm_version is None else chm_version.source
183
+
184
+ # v1.2.0 files have the line "CO source" in the header and pre-v1.2.0 files have no indication
185
+ # of version.
186
+ nhead = mod_utils.get_num_header_lines(mod_file)
187
+ with open(mod_file) as f:
188
+ for idx, line in enumerate(f):
189
+ if idx == nhead:
190
+ # This is the default; .mod files before v1.2.0 did not include a CO
191
+ # source in the header because it was always from GEOS FP-IT. Also
192
+ # prior to v1.2.0, the header only had 7 lines (by default), so if
193
+ # there's >7 lines, that probably means we messed up.
194
+ if nhead > 7:
195
+ logger.warning((f'In .mod file {mod_file}, did not find a "CO source" line in the header, but the header has more than 7 lines. '
196
+ 'Unless this is a custom .mod file, this means I may have missed the CO source line.'))
197
+ return GeosSource.FPIT
198
+ elif line.startswith('CO source'):
199
+ source = line.split(':', maxsplit=2)[1].strip()
200
+ return GeosSource(source)
201
+
202
+
203
+ def read_mod_file_units(mod_file):
204
+ """
205
+ Get the units for the profile variables in a .mod file
206
+
207
+ :param mod_file: the .mod file to read
208
+ :type mod_file: str
209
+
210
+ :return: a dictionary with the variable names as keys and the units as values.
211
+ """
212
+ n_header_lines = mod_utils.get_num_header_lines(mod_file)
213
+ # Assume that the profile units are the second to last line of the header
214
+ # and the profile variable names are the last line
215
+ with open(mod_file, 'r') as robj:
216
+ for i in range(n_header_lines):
217
+ line = robj.readline()
218
+ if i == (n_header_lines-2):
219
+ units = line.split()
220
+ elif i == (n_header_lines-1):
221
+ names = line.split()
222
+
223
+ return {n: u for n, u in zip(names, units)}
224
+
225
+
226
+ def read_map_file(map_file, as_dataframes=False, skip_header=False, fmt=None):
227
+ """
228
+ Read a .map file
229
+
230
+ :param map_file: the path to the .map file
231
+ :type map_file: str
232
+
233
+ :param as_dataframes: set to ``True`` to return the constants and profiles data as Pandas dataframes. By default,
234
+ (``False``) they are returned as dictionaries of numpy arrays.
235
+ :type as_dataframes: bool
236
+
237
+ :param skip_header: set to ``True` to avoid reading the header. This is helpful for reading .map files that
238
+ have a slightly different header format.
239
+ :type skip_header: bool
240
+
241
+ :param fmt: can use to tell this function which file type the .map file is, either "txt" for text or "nc" for
242
+ netCDF. If not specified, it will attempt to infer from the extension.
243
+ :type fmt: str or None
244
+
245
+ :return: a dictionary with keys 'constants' and 'profile' that hold the header values and main profile data,
246
+ respectively. The form of these values depends on ``as_dataframes``.
247
+ :rtype: dict
248
+ """
249
+ if map_file.endswith('.nc') or fmt == 'nc':
250
+ return _read_map_nc_file(map_file, as_dataframes=as_dataframes, skip_header=skip_header)
251
+ elif map_file.endswith('.map') or fmt == 'txt':
252
+ return _read_map_txt_file(map_file, as_dataframes=as_dataframes, skip_header=skip_header)
253
+ else:
254
+ ext = os.path.splitext(map_file)
255
+ raise ValueError('Unknown extension "{}" for map file. Use the `fmt` keyword to indicate the file type.'.format(ext))
256
+
257
+
258
+ def _read_map_nc_file(map_file, as_dataframes=False, skip_header=False):
259
+ profile_dict = dict()
260
+ constant_dict = dict()
261
+ with ncdf.Dataset(map_file) as ds:
262
+ for varname, vardat in ds.variables.items():
263
+ if varname == 'time':
264
+ vardat = ncdf.num2date(vardat[:], vardat.units, only_use_cftime_datetimes=False)
265
+ profile_dict[varname] = pd.DatetimeIndex(vardat)
266
+ else:
267
+ profile_dict[varname] = vardat[:].filled(np.nan)
268
+
269
+ if not skip_header:
270
+ for attr in ds.ncattrs():
271
+ if attr.startswith('constant') and not attr.endswith('units'):
272
+ k = attr.replace('constant_', '')
273
+ constant_dict[k] = ds.getncattr(attr)
274
+
275
+ if as_dataframes:
276
+ # these are both scalar values that can't go into the dataframe
277
+ profile_dict.pop('time')
278
+ profile_dict.pop('lat')
279
+ profile_dict = pd.DataFrame(profile_dict)
280
+ profile_dict.set_index('altitude', inplace=True)
281
+
282
+ constant_dict = pd.DataFrame(constant_dict, index=[0])
283
+
284
+ return {'profile': profile_dict, 'constants': constant_dict}
285
+
286
+
287
+ def _read_map_txt_file(map_file, as_dataframes=False, skip_header=False):
288
+ n_header_lines = mod_utils.get_num_header_lines(map_file)
289
+ constants = dict()
290
+ if not skip_header:
291
+ with open(map_file, 'r') as mapf:
292
+ n_skip = 4
293
+ # Skip the first four lines to get to the constants - these should be (1) the number of header lines &
294
+ # columns, (2) filename, (3) version info, and (4) wiki reference.
295
+ for i in range(n_skip):
296
+ mapf.readline()
297
+
298
+ # The last two lines of the header are the column names and units; everything between line 5 and that should
299
+ # be physical constants. Start at n_skip+1 to account for 0 indexing vs. number of lines.
300
+
301
+ for i in range(n_skip+1, n_header_lines-1):
302
+ line = mapf.readline()
303
+ # Lines have the form Name (units): value - ignore anything in parentheses
304
+ name, value = line.split(':')
305
+ name = re.sub(r'\(.+\)', '', name).strip()
306
+ constants[name] = float(value)
307
+
308
+ df = pd.read_csv(map_file, header=n_header_lines-2, skiprows=[n_header_lines-1], na_values='NAN')
309
+ # Sometimes extra space gets kept in the headers - remove that
310
+ df.rename(columns=lambda h: h.strip(), inplace=True)
311
+ if not as_dataframes:
312
+ data = {k: v.values for k, v in df.items()}
313
+ else:
314
+ data = df
315
+ constants = pd.DataFrame(constants, index=[0])
316
+
317
+ out_dict = dict()
318
+ out_dict['constants'] = constants
319
+ out_dict['profile'] = data
320
+ return out_dict
321
+
322
+
323
+ def read_isotopes(isotopes_file, gases_only=False):
324
+ """
325
+ Read the isotopes defined in an isotopologs.dat file
326
+
327
+ :param isotopes_file: the path to the isotopologs.dat file
328
+ :type isotopes_file: str
329
+
330
+ :param gases_only: set to ``True`` to return a tuple of only the distinct gases, not the individual isotopes.
331
+ Default is ``False``, which includes the different isotope numbers.
332
+ :type gases_only: bool
333
+
334
+ :return: tuple of isotope or gas names
335
+ :rtype: tuple(str)
336
+ """
337
+ nheader = mod_utils.get_num_header_lines(isotopes_file)
338
+ with open(isotopes_file, 'r') as fobj:
339
+ for i in range(nheader):
340
+ fobj.readline()
341
+
342
+ isotopes = []
343
+ for line in fobj:
344
+ iso_number = line[3:5].strip()
345
+ iso_name = line[6:14].strip()
346
+ if not gases_only:
347
+ iso_name = iso_number + iso_name
348
+ if iso_name not in isotopes:
349
+ isotopes.append(iso_name)
350
+
351
+ return tuple(isotopes)
352
+
353
+
354
+ def read_vmr_file(vmr_file, as_dataframes=False, lowercase_names=True, style='new'):
355
+ nheader = mod_utils.get_num_header_lines(vmr_file)
356
+
357
+ if style == 'new':
358
+ last_const_line = nheader - 1
359
+ old_style = False
360
+ elif style == 'old':
361
+ last_const_line = 4
362
+ old_style = True
363
+ else:
364
+ raise ValueError('style must be one of "new" or "old"')
365
+
366
+ header_data = dict()
367
+ with open(vmr_file, 'r') as fobj:
368
+ # Skip the line with the number of header lines and columns
369
+ fobj.readline()
370
+ for i in range(1, last_const_line):
371
+ line = fobj.readline()
372
+ const_name, const_val = [v.strip() for v in line.split(':')]
373
+ if lowercase_names:
374
+ const_name = const_name.lower()
375
+
376
+ try:
377
+ const_val = float(const_val)
378
+ except ValueError:
379
+ pass
380
+ header_data[const_name] = const_val
381
+
382
+ prior_info = dict()
383
+ if old_style:
384
+ for i in range(last_const_line, nheader-1, 2):
385
+ category_line = fobj.readline()
386
+ category = re.split(r'[:\.]', category_line)[0].strip()
387
+ data_line = fobj.readline()
388
+ data_line = data_line.split(':')[1].strip()
389
+ split_data_line = re.split(r'\s+', data_line)
390
+ prior_info[category] = np.array([float(x) for x in split_data_line])
391
+
392
+ data_table = pd.read_csv(vmr_file, sep='\s+', header=nheader-1)
393
+
394
+ # Also get the information that's only in the file name (namely date and longitude, we'll also read the latitude
395
+ # because it's there).
396
+ file_vars = dict()
397
+ base_name = os.path.basename(vmr_file)
398
+ try:
399
+ file_vars['datetime'] = mod_utils.find_datetime_substring(base_name, out_type=dt.datetime)
400
+ file_vars['lon'] = mod_utils.find_lon_substring(base_name, to_float=True)
401
+ file_vars['lat'] = mod_utils.find_lat_substring(base_name, to_float=True)
402
+ except AttributeError:
403
+ # Happens when the regex can't find a date/lon/lat in the file name
404
+ # usually means we're reading a climatological file
405
+ file_vars = dict(datetime=None, lon=None, lat=None)
406
+
407
+ if lowercase_names:
408
+ data_table.columns = [v.lower() for v in data_table]
409
+
410
+ if as_dataframes:
411
+ header_data = pd.DataFrame(header_data, index=[0])
412
+ # Rearrange the prior info dict so that the data frame has the categories as the index and the species as the
413
+ # columns.
414
+ categories = list(prior_info.keys())
415
+ tmp_prior_info = dict()
416
+ for i, k in enumerate(data_table.columns.drop('altitude')):
417
+ tmp_prior_info[k] = np.array([prior_info[cat][i] for cat in categories])
418
+ prior_info = pd.DataFrame(tmp_prior_info, index=categories)
419
+ else:
420
+ # use an ordered dict to ensure we keep the order of the gases. This is important if we use this .vmr file as
421
+ # a template to write another .vmr file that gsetup.f can read.
422
+ data_table = OrderedDict([(k, v.to_numpy()) for k, v in data_table.items()])
423
+
424
+ return {'scalar': header_data, 'profile': data_table, 'prior_info': prior_info, 'file': file_vars}
425
+
426
+
427
+ def read_runlog(runlog_file, as_dataframes=False, remove_commented_lines=True):
428
+ nhead = mod_utils.get_num_header_lines(runlog_file)
429
+ with open(runlog_file, 'r') as robj:
430
+ for i in range(nhead-2):
431
+ # Skip ahead to the last two lines of the header which contain the fortran format specification and the
432
+ # column names
433
+ robj.readline()
434
+
435
+ fmt_line = robj.readline()
436
+ # we need to read the column names twice. read_fwf expects the first line it reads to be the column names but
437
+ # with the right widths. Since that's not the case for the runlog, we read in the line manually once (to parse
438
+ # later into the proper column names) then rewind the file pointed to the beginning of that line so that
439
+ # read_fwf reads it in as the column names. It'll get them wrong, but at least it doesn't stick a data row in
440
+ # as the header.
441
+ pos = robj.tell()
442
+ column_names_line = robj.readline()
443
+ robj.seek(pos)
444
+
445
+ # the format line will be something like: "format=(a1,a57,1x,2i4,f8.4,f8.3,f9.3,...)\n"; we want to pass only
446
+ # the part inside the parentheses (including the parentheses)
447
+ colspecs, _ = mod_utils.fortran_fmt_to_fwf_tuples(fmt_line.split('=')[1].strip())
448
+ df = pd.read_fwf(robj, colspecs=colspecs)
449
+
450
+ # Fix the header. The first column is just where the colon goes if the line is commented out so we have to give
451
+ # that a name (there isn't one in the header)
452
+ column_names = column_names_line.split()
453
+ df.columns = ['comment'] + column_names_line.split()
454
+
455
+ # Remove comments if requested.
456
+ if remove_commented_lines:
457
+ df = df[~(df['comment'].apply(lambda x: x == ':'))]
458
+
459
+ # Always remove the comment column itself
460
+ df = df[column_names]
461
+
462
+ if as_dataframes:
463
+ return df
464
+ else:
465
+ return df.to_dict()
466
+
467
+
468
+ def read_tabular_file_with_header(file_path: Union[str, Path], comment_str: str = '#') -> pd.DataFrame:
469
+ with open(file_path) as f:
470
+ for line in f:
471
+ if not line.startswith(comment_str):
472
+ break
473
+
474
+ columns = line.strip().split()
475
+ df = pd.read_csv(f, header=None, delim_whitespace=True)
476
+ df.columns = columns
477
+ return df
@@ -0,0 +1,57 @@
1
+ import numpy as np
2
+ import pandas as pd
3
+ import sys
4
+
5
+ from . import mod_utils, readers
6
+
7
+
8
+ def iter_runlog_args(runlog, first_date='2000-01-01', site_abbrv=None):
9
+ def _reduce_lat_lons(df):
10
+ slla = []
11
+ for (site, lon, lat), ll_df in df.groupby(['site', 'oblon', 'oblat']):
12
+ if not np.isclose(df['obalt'], df['obalt'][0]).all():
13
+ lon_str = mod_utils.format_lon(lon)
14
+ lat_str = mod_utils.format_lat(lat)
15
+ first_date = ll_df.index.min().strftime('%Y-%m-%d %H:%M:%S')
16
+ last_date = ll_df.index.min().strftime('%Y-%m-%d %H:%M:%S')
17
+ print('WARNING: multiple observation altitudes for {lon}, {lat} in the date range {start} to {stop}. '
18
+ 'Will only produce the .mod files for the lowest altitude. (Note: altitude only matters if '
19
+ 'generating .mod files for a slant path.)'.format(lon=lon_str, lat=lat_str, start=first_date,
20
+ stop=last_date),
21
+ file=sys.stderr)
22
+
23
+ slla.append((site, lon, lat, df['obalt'].min()))
24
+ # convert to lists of sites, lons, lats, and alts separately
25
+ return zip(*slla)
26
+
27
+ rldf = readers.read_runlog(runlog, as_dataframes=True)
28
+ rl_dates = mod_utils.ydh_to_date(rldf['Year'], rldf['Day'], rldf['Hour'])
29
+ rldf.set_index(rl_dates, inplace=True)
30
+
31
+ # GEOS FPIT has no data before 2000 so we usually remove those dates
32
+ if first_date:
33
+ xx_dates = rl_dates >= first_date
34
+ rldf = rldf[xx_dates]
35
+
36
+ date_ranges = mod_utils.get_runlog_geos_date_ranges(rldf)
37
+
38
+ if site_abbrv is None:
39
+ # If no site abbreviation given, assume that the site abbreviations are the first two letters of each spectrum
40
+ site_abbrv = [s[:2] for s in rldf['Spectrum_File_Name']]
41
+ elif isinstance(site_abbrv, str):
42
+ # If a single site abbreviation given, then it must be used for all sites, so we need to expand it so that we
43
+ # can create a date-indexed series to subset as we loop through the date ranges
44
+ site_abbrv = rldf.shape[0] * [site_abbrv]
45
+
46
+ rldf['site'] = site_abbrv
47
+
48
+ for drange in date_ranges:
49
+ dstart = (drange[0] - pd.Timedelta(minutes=90)).strftime('%Y-%m-%d %H:%M:%S')
50
+ dstop = drange[1].strftime('%Y-%m-%d %H:%M:%S') # end time is already one GEOS time step ahead
51
+ xx = slice(dstart, dstop)
52
+ sub_df = rldf.loc[xx, ['oblon', 'oblat', 'obalt', 'site']]
53
+ if sub_df.shape[0] == 0:
54
+ raise RuntimeError('Found no runlog entries between {} and {}; error in calculated date ranges?'
55
+ .format(dstart, dstop))
56
+ abbrv, lon, lat, alt = _reduce_lat_lons(sub_df)
57
+ yield drange, abbrv, lon, lat, alt
@@ -0,0 +1,67 @@
1
+ import numpy as np
2
+
3
+
4
+ def time_weight_from_datetime(acos_datetime, prev_geos_datetime, next_geos_datetime):
5
+ """
6
+ Calculate the time weighting from datetime-like objects.
7
+
8
+ :param acos_datetime: the date/time of the ACOS sounding
9
+ :type acos_datetime: datetime-like
10
+
11
+ :param prev_geos_datetime: the date/time of the GEOS FP file that comes before the ACOS sounding
12
+ :type prev_geos_datetime: datetime-like
13
+
14
+ :param next_geos_datetime: the date/time of the GEOS FP file that comes after the ACOS sounding
15
+ :type next_geos_datetime: datetime-like
16
+
17
+ :return: the weight, w, such that the profile for the ACOS time would be :math:`w*p1 + (1-w)*p2`, where p1 and p2
18
+ are the profiles at the GEOS times before and after the sounding, respectively.
19
+ :rtype: float
20
+ """
21
+ return time_weight(datetime2datenum(acos_datetime), datetime2datenum(prev_geos_datetime), datetime2datenum(next_geos_datetime))
22
+
23
+
24
+ def time_weight(acos_datenum, prev_geos_datenum, next_geos_datenum):
25
+ """
26
+ Calculate the time weighting from date numbers.
27
+
28
+ In this context, a date number is any representation of a date time as a integer- or float-like value. Typically,
29
+ this will be a value returned by :func:`datetime2datenum` but any similar numeric value will do, so long as all
30
+ inputs are consistent.
31
+
32
+ :param acos_datenum: the date/time of the ACOS sounding as a number
33
+ :type acos_datenum: int or float
34
+
35
+ :param prev_geos_datenum: the date/time of the GEOS FP file that comes before the ACOS sounding as a number
36
+ :type prev_geos_datenum: int or float
37
+
38
+ :param next_geos_datenum: the date/time of the GEOS FP file that comes after the ACOS sounding as a number
39
+ :type next_geos_datenum: int or float
40
+
41
+ :return: the weight, w, such that the profile for the ACOS time would be :math:`w*p1 + (1-w)*p2`, where p1 and p2
42
+ are the profiles at the GEOS times before and after the sounding, respectively.
43
+ :rtype: float
44
+ """
45
+ # if acos_datenum == prev_geos_datenum, then this == 1 => p = 1*p1 + 0*p2 = p1
46
+ # if acos_datenum == next_geos_datenum, then this == 0 => p = 0*p1 + 1*p2 = p2
47
+ return (next_geos_datenum - acos_datenum) / (next_geos_datenum - prev_geos_datenum)
48
+
49
+
50
+ def datetime2datenum(datetime_obj):
51
+ """
52
+ Convert a single :class:`datetime.datetime` object into a date number.
53
+
54
+ Internally, this converts the datetime object into a :class:`numpy.datetime64` object with units of seconds, then
55
+ converts that to a float type. Under numpy version 1.16.2, this results in a number that is seconds since 1 Jan
56
+ 1970; however, I have not seen any documentation from numpy guaranteeing that behavior. Therefore, any use of these
57
+ date numbers should be careful to verify this behavior. The following assert block can be used to check this::
58
+
59
+ assert numpy.isclose(datetime2datenum('1970-01-01'), 0.0) and numpy.isclose(datetime2datenum('1970-01-02'), 86400)
60
+
61
+ :param datetime_obj: the datetime to convert. May be any type that :class:`numpy.datetime64` can intepret as a
62
+ datetime.
63
+
64
+ :return: the converted date number
65
+ :rtype: :class:`numpy.float`
66
+ """
67
+ return np.datetime64(datetime_obj, 's').astype(float)