ginput 1.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. ginput/__init__.py +4 -0
  2. ginput/common_utils/__init__.py +0 -0
  3. ginput/common_utils/ggg_logging.py +62 -0
  4. ginput/common_utils/ioutils.py +304 -0
  5. ginput/common_utils/mod_constants.py +24 -0
  6. ginput/common_utils/mod_utils.py +2663 -0
  7. ginput/common_utils/readers.py +477 -0
  8. ginput/common_utils/run_utils.py +57 -0
  9. ginput/common_utils/sat_utils.py +67 -0
  10. ginput/common_utils/versioning.py +251 -0
  11. ginput/common_utils/writers.py +377 -0
  12. ginput/data/.gitignore +1 -0
  13. ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
  14. ginput/data/ML_monthly_obs_ch4.txt +498 -0
  15. ginput/data/ML_monthly_obs_co.txt +412 -0
  16. ginput/data/ML_monthly_obs_co2.txt +597 -0
  17. ginput/data/ML_monthly_obs_n2o.txt +330 -0
  18. ginput/data/SMO_monthly_obs_ch4.txt +499 -0
  19. ginput/data/SMO_monthly_obs_co.txt +416 -0
  20. ginput/data/SMO_monthly_obs_co2.txt +613 -0
  21. ginput/data/SMO_monthly_obs_n2o.txt +330 -0
  22. ginput/data/ace_fn2o_lut.nc +0 -0
  23. ginput/data/age_spectra/age.midlat.txt +47 -0
  24. ginput/data/age_spectra/age.tropics.txt +46 -0
  25. ginput/data/age_spectra/age.vortex.txt +31 -0
  26. ginput/data/age_spectra/agespec.midlat.txt +47 -0
  27. ginput/data/age_spectra/agespec.tropics.txt +46 -0
  28. ginput/data/age_spectra/agespec.vortex.txt +31 -0
  29. ginput/data/age_spectra/time.txt +3001 -0
  30. ginput/data/ch4_hf_slopes.nc +0 -0
  31. ginput/data/ch4_strat_lut.nc +0 -0
  32. ginput/data/clams_age_clim_scaled.nc +0 -0
  33. ginput/data/co2_strat_lut.nc +0 -0
  34. ginput/data/fo2_data/.gitignore +2 -0
  35. ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
  36. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
  37. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
  38. ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
  39. ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
  40. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
  41. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
  42. ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
  43. ginput/data/hf_strat_lut.nc +0 -0
  44. ginput/data/isotopologs.dat +199 -0
  45. ginput/data/meso_co_lut.nc +0 -0
  46. ginput/data/n2o_ch4_acefts.nc +0 -0
  47. ginput/data/n2o_strat_lut.nc +0 -0
  48. ginput/data/o2_mean_dmf.dat +52 -0
  49. ginput/data/summer_35N.vmr +182 -0
  50. ginput/data/washenfelder03_table_s3.txt +83 -0
  51. ginput/download/__init__.py +0 -0
  52. ginput/download/download_utils.py +46 -0
  53. ginput/download/get_GEOS5.py +367 -0
  54. ginput/download/get_MERRA2.py +177 -0
  55. ginput/download/get_NOAA_flask_data.py +103 -0
  56. ginput/download/get_fo2_data.py +140 -0
  57. ginput/mod_maker/__init__.py +0 -0
  58. ginput/mod_maker/mod_maker.py +2470 -0
  59. ginput/mod_maker/slantify.py +450 -0
  60. ginput/mod_maker/tccon_sites.py +570 -0
  61. ginput/priors/__init__.py +0 -0
  62. ginput/priors/acos_interface.py +1376 -0
  63. ginput/priors/automation.py +313 -0
  64. ginput/priors/backend_analysis/__init__.py +0 -0
  65. ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
  66. ginput/priors/backend_analysis/ace_fts_val.py +452 -0
  67. ginput/priors/backend_analysis/acos_comparison.py +550 -0
  68. ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
  69. ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
  70. ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
  71. ginput/priors/backend_analysis/backend_driver.py +90 -0
  72. ginput/priors/backend_analysis/backend_utils.py +358 -0
  73. ginput/priors/backend_analysis/clams.py +71 -0
  74. ginput/priors/backend_analysis/create_test_priors.py +853 -0
  75. ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
  76. ginput/priors/backend_analysis/mls_val.py +217 -0
  77. ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
  78. ginput/priors/fo2_prep.py +311 -0
  79. ginput/priors/map_maker.py +193 -0
  80. ginput/priors/mlo_smo_prep.py +1146 -0
  81. ginput/priors/tccon_priors.py +3482 -0
  82. ginput-1.3.0.dist-info/LICENSE.txt +203 -0
  83. ginput-1.3.0.dist-info/METADATA +125 -0
  84. ginput-1.3.0.dist-info/RECORD +86 -0
  85. ginput-1.3.0.dist-info/WHEEL +5 -0
  86. ginput-1.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2663 @@
1
+ """
2
+ Utility functions shared among the mod maker and prior generation code.
3
+
4
+ Compatibility notes:
5
+ * We have found that Pandas version 0.20 is incompatible with this module. Pandas >= 0.24 works.
6
+ """
7
+
8
+ from __future__ import print_function, division
9
+
10
+ import datetime as dt
11
+ from datetime import timedelta
12
+ from glob import glob
13
+ from hashlib import md5
14
+
15
+ import xarray as xr
16
+ from dateutil.relativedelta import relativedelta
17
+ import netCDF4 as ncdf
18
+ import numpy as np
19
+ from numpy import ma
20
+ import os
21
+ import pandas as pd
22
+ import re
23
+
24
+ from numpy.core._multiarray_umath import arctan, tan, sin, cos
25
+ from scipy.interpolate import interp1d, interp2d
26
+ import subprocess
27
+ import sys
28
+ from warnings import warn
29
+
30
+ from . import mod_constants as const
31
+ from .mod_constants import days_per_year
32
+ from ..mod_maker.tccon_sites import site_dict
33
+
34
+ _std_model_pres_levels = np.array([1000.0, 975.0, 950.0, 925.0, 900.0, 875.0, 850.0, 825.0, 800.0, 775.0, 750.0, 725.0,
35
+ 700.0, 650.0, 600.0, 550.0, 500.0, 450.0, 400.0, 350.0, 300.0, 250.0, 200.0, 150.0,
36
+ 100.0, 70.0, 50.0, 40.0, 30.0, 20.0, 10.0, 7.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.7, 0.5,
37
+ 0.4, 0.3, 0.1])
38
+ # as of 2019-03-14, both GEOS-FP and MERRA-2 use the same standard 42 pressure levels
39
+ # (c.f. https://gmao.gsfc.nasa.gov/GMAO_products/documents/GEOS_5_FP_File_Specification_ON4v1_2.pdf page 52 and
40
+ # https://gmao.gsfc.nasa.gov/reanalysis/MERRA/docs/MERRA_File_Specification.pdf page 27 for GEOS-FP and MERRA-2
41
+ # respectively)
42
+ merra_pres_levels = _std_model_pres_levels
43
+ geosfp_pres_levels = _std_model_pres_levels
44
+ earth_radius = 6371 # kilometers
45
+
46
+
47
+ class TropopauseError(Exception):
48
+ """
49
+ Error if could not find the tropopause
50
+ """
51
+ pass
52
+
53
+
54
+ class ModelError(Exception):
55
+ """
56
+ Error if a model file is nonsensical.
57
+ """
58
+ pass
59
+
60
+
61
+ class GGGPathError(Exception):
62
+ pass
63
+
64
+
65
+ class ModPathError(IOError):
66
+ pass
67
+
68
+
69
+ class ProgressBar(object):
70
+ """
71
+ Create a text-based progress bar
72
+
73
+ An instance of this class can be used to print a text progress bar that does not need a new line for each progress
74
+ step. It uses carriage returns to reset to the beginning of each line before printing the next. This therefore
75
+ does not work well if other print statements occur in between calls to :meth:`print_bar`, the progress bar will
76
+ either end up on a new line anyway or potentially overwrite previous print statements if they did not end with a
77
+ newline.
78
+
79
+ :param num_symbols: how many steps there should be in the progress bar. In other words, the progress bar will be
80
+ complete when :meth:`print_bar` is called with ``num_symbols-1``.
81
+ :type num_symbols: int
82
+
83
+ :param prefix: a string to include before the beginning of each progress bar. The class will ensure that at least
84
+ one space is present between the prefix and the progress bar, but will not add one if one is already present at
85
+ the end of the prefix.
86
+ :type prefix: str
87
+
88
+ :param suffix: a string to include at the end of each progress bar. The class will ensure that at least one space
89
+ is present between the progress bar and the suffix, but will not add one if one is already present at the beginning
90
+ of the suffix.
91
+ :type suffix: str
92
+
93
+ :param add_one: if ``True``, the number of symbols printed in the progress bar is equal to ``i+1`` where ``i`` is
94
+ the argument to :meth:`print_bar`. This works well with Python loops over ``i in range(n)``, since the last value
95
+ of ``i`` will be ``n-1``, setting ``add_one`` to ``True`` ensures that a full progress bar is printed at the end.
96
+ :type add_one: bool
97
+
98
+ :param style: can be either '*' or 'counter'. The former prints a symbolic progress bar of the form:
99
+
100
+ [* ]
101
+ [** ]
102
+ [*** ]
103
+ [****]
104
+
105
+ where the number of *'s is set by ``num_symbols``. The latter will instead print 'i/num_symbols' for each step.
106
+ :type style: str
107
+ """
108
+ def __init__(self, num_symbols, prefix='', suffix='', add_one=True, style='*'):
109
+ """
110
+ See class help.
111
+ """
112
+ if len(prefix) > 0 and not prefix.endswith(' '):
113
+ prefix += ' '
114
+ if len(suffix) > 0 and not suffix.startswith(' '):
115
+ suffix = ' ' + suffix
116
+
117
+ if style == '*':
118
+ self._fmt_str = '{pre}[{{pstr:<{n}}}]{suf}'.format(pre=prefix, n=num_symbols, suf=suffix)
119
+ elif style == 'counter':
120
+ self._fmt_str = '{pre}{{i:>{l}}}/{n}{suf}'.format(pre=prefix, n=num_symbols, suf=suffix, l=len(str(num_symbols)))
121
+ else:
122
+ raise ValueError('style "{}" not recognized'.format(style))
123
+ self._add_one = add_one
124
+
125
+ def print_bar(self, i):
126
+ """
127
+ Print the iteration of the progress bar corresponding to step ``i``.
128
+
129
+ :param i: defines the progress step, either the number of *'s to print with ``style='*'`` or the counter number
130
+ with ``style='counter'``.
131
+ :type i: int
132
+ :return: None, prints to screen.
133
+ """
134
+ if self._add_one:
135
+ i += 1
136
+
137
+ pstr = '*' * i
138
+ pbar = self._fmt_str.format(pstr=pstr, i=i)
139
+ sys.stdout.write('\r' + pbar)
140
+ sys.stdout.flush()
141
+
142
+ def finish(self):
143
+ """
144
+ Close the progress bar. By default, just prints a newline.
145
+ :return: None
146
+ """
147
+ sys.stdout.write('\n')
148
+ sys.stdout.flush()
149
+
150
+
151
+ def check_depedencies_newer(out_file, *dependency_files):
152
+ """
153
+ Check if
154
+ :param out_file:
155
+ :param dependency_files:
156
+ :return:
157
+ """
158
+ if len(dependency_files) == 0:
159
+ raise ValueError('Give at least one dependency file')
160
+
161
+ out_last_modified = os.path.getmtime(out_file)
162
+ for dep in dependency_files:
163
+ if os.path.getmtime(dep) > out_last_modified:
164
+ return True
165
+
166
+ return False
167
+
168
+
169
+ def get_num_header_lines(filename):
170
+ """
171
+ Get the number of header lines in a standard GGG file
172
+
173
+ This assumes that the file specified begins with a line with two numbers: the number of header rows and the number
174
+ of data columns.
175
+
176
+ :param filename: the file to read
177
+ :type filename: str
178
+
179
+ :return: the number of header lines
180
+ :rtype: int
181
+ """
182
+ with open(filename, 'r') as fobj:
183
+ header_info = fobj.readline()
184
+
185
+ if ',' in header_info:
186
+ header = header_info.split(',')
187
+ else:
188
+ header = header_info.split()
189
+ return int(header[0])
190
+
191
+
192
+ def get_isotopes_file(isotopes_file=None, use_gggpath=False):
193
+ """
194
+ Get the path to the isotopologs.dat file
195
+
196
+ :param isotopes_file: user input path. If this is not None, it is returned after checking that it exists.
197
+ :type isotopes_file: str or None
198
+
199
+ :param use_gggpath: set to ``True`` to find the isotopologs.dat file at the location defined by the GGGPATH
200
+ environmental variable. If ``False`` and ``isotopes_file`` is ``None`` then the isotopologs.date file included in
201
+ this repo is used.
202
+ :type use_gggpath: bool
203
+
204
+ :return: the path to the isotopologs.dat file
205
+ :rtype: str
206
+ """
207
+ if isotopes_file is not None:
208
+ if not os.path.isfile(isotopes_file):
209
+ raise IOError('The isotopes path {} is not a file'.format(isotopes_file))
210
+ return isotopes_file
211
+ elif use_gggpath:
212
+ gggpath = os.getenv('GGGPATH')
213
+ if gggpath is None:
214
+ raise EnvironmentError('use_gggpath=True requires the GGGPATH environmental variable to be set')
215
+ isotopes_file = os.path.join(gggpath, 'isotopologs', 'isotopologs.dat')
216
+ if not os.path.isfile(isotopes_file):
217
+ raise IOError('Failed to find isotopologs.dat at {}. Either update your GGGPATH environmental variable or '
218
+ 'set use_gggpath to False.'.format(isotopes_file))
219
+ return isotopes_file
220
+ else:
221
+ return os.path.join(const.data_dir, 'isotopologs.dat')
222
+
223
+
224
+ def mod_output_subdir(top_dir, site_abbrev, product='fpit', slant=False):
225
+ path_dir = 'slant' if slant else 'vertical'
226
+ return os.path.join(top_dir, product, site_abbrev, path_dir)
227
+
228
+
229
+ def vmr_output_subdir(top_dir, site_abbrev, product='fpit', slant=False):
230
+ path_dir = 'vmrs-slant' if slant else 'vmrs-vertical'
231
+ return os.path.join(top_dir, product, site_abbrev, path_dir)
232
+
233
+
234
+ def vmr_file_name(obs_date, lon, lat, keep_latlon_prec=False, date_fmt='%Y%m%d%H', in_utc=True):
235
+ """
236
+ Construct the standard filename for a .vmr file produced by this code
237
+
238
+ :param obs_date: the datetime of the profiles
239
+ :type obs_date: datetime-like
240
+
241
+ :param lon: the longitude of the profiles.
242
+ :type lon: float
243
+
244
+ :param lat: the latitude of the profiles
245
+ :type lat: float
246
+
247
+ :param keep_latlon_prec: by default, lat and lon are rounded to the nearest whole number. Set this to ``True`` to
248
+ keep 2 decimal places of precision.
249
+ :type keep_latlon_prec: bool
250
+
251
+ :return: the .vmr file name, with format "JLv_yyyymmddhh_XX[NS]_YYY[EW].vmr" where "v" is the major version,
252
+ "yyyymmddhh" the date/time, XX[NS] the latitude and YYY[EW] the longitude.
253
+ :rtype: str
254
+ """
255
+ # The desired behavior is to zero pad if rounding to whole degrees but not otherwise. This mimics the behavior for
256
+ # .mod files, although that function does it slightly differently, and reconciling the two should be done at some
257
+ # point.
258
+ prec = 2 if keep_latlon_prec else 0
259
+ pad = False if keep_latlon_prec else True
260
+ lat = format_lat(lat, prec=prec, zero_pad=pad)
261
+ lon = format_lon(lon, prec=prec, zero_pad=pad)
262
+ major_version = _get_priors_major_version()
263
+ return 'JL{ver}_{date}{tz}_{lat}_{lon}.vmr'.format(ver=major_version, date=obs_date.strftime(date_fmt),
264
+ tz='Z' if in_utc else 'L', lat=lat, lon=lon)
265
+
266
+
267
+ def vmr_file_name_from_mod(mod_file_name: str) -> str:
268
+ """
269
+ Infer the base file name for a .vmr file given the corresponding .mod file.
270
+
271
+ :param mod_file_name: The base name (NOT the full path) of the .mod file. This must have the
272
+ where everything after the first underscore and leading up to the extension is shared with
273
+ the .vmr file name.
274
+
275
+ :returns: the base name for the .vmr file.
276
+ """
277
+ # Assume the mod file name is something like "FPIT_2021042300Z_52N_000W.mod" and we want
278
+ # something like "JL1_2021042300Z_52N_000W.vmr". All we need do is replace the part before
279
+ # the first underscore.
280
+ date_loc = mod_file_name.split('_', maxsplit=1)[1].replace('.mod', '')
281
+ return 'JL{}_{}.vmr'.format(_get_priors_major_version(), date_loc)
282
+
283
+
284
+ def _get_priors_major_version():
285
+ return const.priors_version.split('.')[0]
286
+
287
+
288
+ def map_file_name_from_mod_vmr_files(site_abbrev, mod_file, vmr_file, map_fmt):
289
+ if map_fmt not in {'nc', 'netcdf', 'txt', 'text'}:
290
+ raise ValueError(f'Unknown value for map_fmt: {map_fmt}')
291
+ is_ncdf = map_fmt in {'nc', 'netcdf'}
292
+
293
+ vmr_date = find_datetime_substring(os.path.basename(vmr_file), out_type=dt.datetime)
294
+ mod_date = find_datetime_substring(os.path.basename(mod_file), out_type=dt.datetime)
295
+ if vmr_date != mod_date:
296
+ raise RuntimeError('The .vmr and .mod files have different dates in their filenames!')
297
+ vmr_lonstr = find_lon_substring(os.path.basename(vmr_file))
298
+ mod_lonstr = find_lon_substring(os.path.basename(mod_file))
299
+ if vmr_lonstr != mod_lonstr:
300
+ raise RuntimeError('The .vmr and .mod files have different longitudes in their filenames!')
301
+ vmr_latstr = find_lat_substring(os.path.basename(vmr_file))
302
+ mod_latstr = find_lat_substring(os.path.basename(mod_file))
303
+ if vmr_latstr != mod_latstr:
304
+ raise RuntimeError('The .vmr and .mod files have different latitudes in their filenames!')
305
+
306
+ return '{site}_{lat}_{lon}_{date}Z.map{suffix}'.format(
307
+ site=site_abbrev, lat=vmr_latstr, lon=vmr_lonstr,
308
+ date=vmr_date.strftime('%Y%m%d%H'),
309
+ suffix='.nc' if is_ncdf else ''
310
+ )
311
+
312
+
313
+ def format_lon(lon, prec=2, zero_pad=False):
314
+ """
315
+ Convert longitude between string and numeric representations.
316
+
317
+ If ``lon`` is a number, then it is converted to a string. The string will be the absolute value with "W" or "E" at
318
+ the end to indicate west (<= 0) or east (> 0). If given a string in that format, it converts it to a number.
319
+
320
+ :param lon: the longitude to convert
321
+ :type lon: float or str
322
+
323
+ :param prec: the precision after the decimal point to use. Only has an effect when converting float to string.
324
+ :type prec: int
325
+
326
+ :param zero_pad: set to ``True`` to zero pad the longitude string so that there are 3 digits before the decimal
327
+ place. Only has an effect when converting float to string.
328
+
329
+ :return: the formatted longitude string or the float representation of the longitude, with west being negative.
330
+ :rtype: str or float
331
+ """
332
+ def to_str(lon):
333
+ ew = 'E' if lon >= 0 else 'W'
334
+ # In Python float format specification, "0X.Y" means to zero pad so that there's X total characters and Y after
335
+ # the decimal point. We want lon zero padded to have three numbers before the decimal point, so the total width
336
+ # needs to be the precision + 4 if there will be a decimal point otherwise just 3.
337
+ width = prec + 4 if prec > 0 else prec + 3
338
+ pad = '0{}'.format(width) if zero_pad else ''
339
+ fmt_str = '{{:{padding}.{prec}f}}{{}}'.format(padding=pad, prec=prec)
340
+ return fmt_str.format(abs(lon), ew)
341
+
342
+ def to_float(lon):
343
+ if lon[-1] == 'E':
344
+ sign = 1
345
+ elif lon[-1] == 'W':
346
+ sign = -1
347
+ else:
348
+ raise ValueError('A longitude string must end in "E" or "W"')
349
+
350
+ return float(lon[:-1]) * sign
351
+
352
+ if isinstance(lon, str):
353
+ return to_float(lon)
354
+ else:
355
+ if lon > 180:
356
+ lon -= 360
357
+ return to_str(lon)
358
+
359
+
360
+ def find_lon_substring(string, to_float=False):
361
+ """
362
+ Find a longitude substring in a string.
363
+
364
+ A longitude substring will match \d+[EW] or \d+\.\d+[EW].
365
+
366
+ :param string: the string to search for the longitude substring
367
+ :type string: str
368
+
369
+ :param to_float: when ``True``, converts the longitude to a float value using :func:`format_lon`, else returns the
370
+ string itself.
371
+ :type to_float: bool
372
+
373
+ :return: the longitude substring or float value
374
+ :rtype: str or float
375
+ """
376
+ # search for one or more numbers, which may include a decimal point followed by at least one number then E or W.
377
+ lon_re = r'\d+(\.\d+)?[EW]'
378
+ lon_str = re.search(lon_re, string).group()
379
+ if to_float:
380
+ return format_lon(lon_str)
381
+ else:
382
+ return lon_str
383
+
384
+
385
+ def format_lat(lat, prec=2, zero_pad=False):
386
+ """
387
+ Convert latitude between string and numeric representations.
388
+
389
+ If ``lat`` is a number, then it is converted to a string. The string will be the absolute value with "N" or "S" at
390
+ the end to indicate south (<= 0) or north (> 0). If given a string in that format, it converts it to a number.
391
+
392
+ :param lat: the latitude to convert
393
+ :type lat: float or str
394
+
395
+ :param prec: the precision after the decimal point to use. Only has an effect when converting float to string.
396
+ :type prec: int
397
+
398
+ :param zero_pad: set to ``True`` to zero pad the latitude string so that there are 2 digits before the decimal
399
+ place. Only has an effect when converting float to string.
400
+
401
+ :return: the formatted latitude string or the float representation of the latitude, with south being negative.
402
+ :rtype: str or float
403
+ """
404
+ def to_str(lat):
405
+ ns = 'N' if lat >= 0 else 'S'
406
+ # In Python float format specification, "0X.Y" means to zero pad so that there's X total characters and Y after
407
+ # the decimal point. We want lat zero padded to have two numbers before the decimal point, so the total width
408
+ # needs to be the precision + 3 if there will be a decimal point otherwise just 2.
409
+ width = prec + 3 if prec > 0 else prec + 2
410
+ pad = '0{}'.format(width) if zero_pad else ''
411
+ fmt_str = '{{:{padding}.{prec}f}}{{}}'.format(padding=pad, prec=prec)
412
+ return fmt_str.format(abs(lat), ns)
413
+
414
+ def to_float(lat):
415
+ if lat[-1] == 'N':
416
+ sign = 1
417
+ elif lat[-1] == 'S':
418
+ sign = -1
419
+ else:
420
+ raise ValueError('A latitude string must end in "N" or "S"')
421
+
422
+ return float(lat[:-1]) * sign
423
+
424
+ if isinstance(lat, str):
425
+ return to_float(lat)
426
+ else:
427
+ return to_str(lat)
428
+
429
+
430
+ def find_lat_substring(string, to_float=False):
431
+ """
432
+ Find a latitude substring in a string.
433
+
434
+ A latitude substring will match \d+[NS] or \d+\.\d+[NS].
435
+
436
+ :param string: the string to search for the latitude substring
437
+ :type string: str
438
+
439
+ :param to_float: when ``True``, converts the latitude to a float value using :func:`format_lat`, else returns the
440
+ string itself.
441
+ :type to_float: bool
442
+
443
+ :return: the latitude substring or float value
444
+ :rtype: str or float
445
+ """
446
+ # search for one or more numbers, which may include a decimal point followed by at least one number then N or S.
447
+ lat_re = r'\d+(\.\d+)?[NS]'
448
+ lat_str = re.search(lat_re, string).group()
449
+ if to_float:
450
+ return format_lat(lat_str)
451
+ else:
452
+ return lat_str
453
+
454
+
455
+ def find_datetime_substring(string, out_type=str):
456
+ """
457
+ Extract a date/time substring from a string.
458
+
459
+ This assumes that the date/time is formatted as %Y%m%d (YYYYMMDD) or %Y%m%d_%H%M (YYYYMMDD_hhmm).
460
+
461
+ :param string: the string to search for the date/time substring.
462
+ :type string: str
463
+
464
+ :param out_type: what type to return the date/time as. Default is to return the string. If another type is passed,
465
+ then it must have a ``strptime`` class method that accepts the string to parse and the format string as arguments,
466
+ i.e. it must behave like :func:`datetime.datetime.strptime`.
467
+ :type out_type: type
468
+
469
+ :return: the string or parsed datetime value
470
+ """
471
+
472
+ # Search for YYYYMMDDHH or YYYYMMDD_HHMM preferentially but allow just YYYYMMDD
473
+ date_re = r'\d{8}(_\d{4}|\d{2})?'
474
+ date_str = re.search(date_re, string).group()
475
+
476
+ if out_type is str:
477
+ return date_str
478
+ else:
479
+ date_fmts = {8: '%Y%m%d', 10: '%Y%m%d%H', 13: '%Y%m%d_%H%M'}
480
+ date_fmt = date_fmts[len(date_str)]
481
+ return out_type.strptime(date_str, date_fmt)
482
+
483
+
484
+ def _vcs_dir_helper(vcs_dir):
485
+ if vcs_dir is None:
486
+ vcs_dir = os.path.dirname(__file__)
487
+ if len(vcs_dir) == 0:
488
+ # If in the current directory, then dirname(__file__) gives an empty string, which isn't allowed as the argument
489
+ # to cwd in check_output
490
+ vcs_dir = '.'
491
+ return os.path.abspath(vcs_dir)
492
+
493
+
494
+ def _is_git_repo(vcs_dir=None):
495
+ vcs_dir = _vcs_dir_helper(vcs_dir)
496
+ try:
497
+ subprocess.check_call(['git', 'status'], cwd=vcs_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
498
+ except subprocess.CalledProcessError:
499
+ return False
500
+ else:
501
+ return True
502
+
503
+ def vcs_commit_info(vcs_dir=None):
504
+ """
505
+ Return the last commit hash on the current branch, current branch, and last commit date for a VCS repo.
506
+
507
+ This function will use Git if ``vcs_dir`` points to a valid Git repo and only fall back
508
+ on Mercurial if ``vcs_dir`` is not a Git repo.
509
+
510
+ :param vcs_dir: optional, the VCS directory to check. If not given, defaults to the one containing this repo.
511
+ :type vcs_dir: str
512
+
513
+ :return: the commit hash, branch name, and date. The date will be in the format e.g. "Mon Aug 01 18:20:15 2023 -0700",
514
+ i.e. "%a %b %d %H:%M:%S %Y %z" format in terms of strftime codes.
515
+ :rtype: str, str, str
516
+ """
517
+ if _is_git_repo(vcs_dir):
518
+ return _git_commit_info(vcs_dir)
519
+ else:
520
+ return _hg_commit_info(vcs_dir)
521
+
522
+
523
+ def _git_commit_info(git_dir=None):
524
+ git_dir = _vcs_dir_helper(git_dir)
525
+ # Get the last commit in the current branch
526
+ parent = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=git_dir).decode('utf8').strip()
527
+ # Get the current branch
528
+ branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=git_dir).decode('utf8').strip()
529
+ # And the date of the last commit. Git by default doesn't seem to zero pad the date,
530
+ # so force it to in order to match Mercurial
531
+ date = subprocess.check_output(['git', 'log', '-1', r'--format=%cd', r'--date=format:"%a %b %d %H:%M:%S %Y %z"'], cwd=git_dir).decode('utf8').strip()
532
+
533
+ return parent, branch, date
534
+
535
+
536
+ def _hg_commit_info(hg_dir=None):
537
+ hg_dir = _vcs_dir_helper(hg_dir)
538
+
539
+ # Get the last commit (-l 1) in the current branch (-f)
540
+ summary = subprocess.check_output(['hg', 'log', '-f', '-l', '1'], cwd=hg_dir).splitlines()
541
+ log_dict = dict()
542
+ # Since subprocess returns a bytes object (at least on Linux) rather than an encoded string object, all the strings
543
+ # below must be bytes, not unicode strings
544
+ for line in summary:
545
+ splitline = line.split(b':', 1)
546
+ if len(splitline) < 2:
547
+ continue
548
+ k, v = splitline
549
+ log_dict[k.strip()] = v.strip()
550
+
551
+ parent = re.search(b'(?<=:)\\w+', log_dict[b'changeset']).group()
552
+ # In Mercurial, if on the default branch, then log does not include a branch name in the output
553
+ branch = log_dict[b'branch'] if b'branch' in log_dict else b'default'
554
+ parent_date = log_dict[b'date']
555
+ # Convert to unicode strings to avoid them getting formatted as "b'abc'" or "b'default'" in unicode strings
556
+ return parent.decode('utf8'), branch.decode('utf8'), parent_date.decode('utf8')
557
+
558
+ def vcs_is_commit_clean(vcs_dir=None, ignore_untracked=True, ignore_files=tuple()):
559
+ """
560
+ Checks if a VCS directory is clean.
561
+
562
+ By default, a directory is considered clean if all tracked files have no uncommitted changes. Untracked files are
563
+ not considered. Setting ``ignore_untracked`` to ``False`` means that there must be no untracked files for the
564
+ directory to be clean. This function will use Git if ``vcs_dir`` points to a valid Git repo and only fall back
565
+ on Mercurial if ``vcs_dir`` is not a Git repo.
566
+
567
+ :param vcs_dir: optional, the VCS directory to check. If not given, defaults to the one containing this repo.
568
+ :type vcs_dir: str
569
+
570
+ :param ignore_untracked: optional, set to ``False`` to require that there be no untracked files in the directory for
571
+ it to be considered clean.
572
+ :type ignore_untracked: bool
573
+
574
+ :param ignore_files: optional, a sequence of file paths (absolute or relative to the current working directory, not the
575
+ repo root) to ignore for the sake of determining if the repo is clean.
576
+ :type ignore_files: Sequence[str]
577
+
578
+ :return: ``True`` if the directory is clean, ``False`` otherwise.
579
+ :rtype: bool
580
+ """
581
+ if _is_git_repo(vcs_dir):
582
+ return _git_is_commit_clean(vcs_dir, ignore_untracked=ignore_untracked, ignore_files=ignore_files)
583
+ else:
584
+ return _hg_is_commit_clean(vcs_dir, ignore_untracked=ignore_untracked, ignore_files=ignore_files)
585
+
586
+ def _in_vcs_ignore(f: str, root: str, ignore_files):
587
+ """Returns True if ``f`` was in ``ignored_files``
588
+
589
+ ``f`` must be a string that is a file path relative to ``root``, which is
590
+ the root of the VCS repo. ``ignore_files`` then is a sequence of strings
591
+ giving the list of files to ignore if they have changed since the last commit.
592
+ Note that ``ignore_files`` will *not* be relative to ``root``, they must be
593
+ absolute paths or relative to the current working directory.
594
+ """
595
+ f = os.path.join(root, f)
596
+ for ignore in ignore_files:
597
+ if os.path.exists(ignore) and os.path.samefile(f, ignore):
598
+ return True
599
+ return False
600
+
601
+ def _git_is_commit_clean(git_dir=None, ignore_untracked=True, ignore_files=tuple()):
602
+ git_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode('utf8').strip()
603
+ summary = subprocess.check_output(['git', 'status', '--porcelain']).decode('utf8').strip().splitlines()
604
+ for line in summary:
605
+ stat, filename = line.split(maxsplit=2)
606
+ if stat == '??' and ignore_untracked:
607
+ continue
608
+ elif stat == '!!':
609
+ # ignored file according to the git status manpage?
610
+ continue
611
+ elif _in_vcs_ignore(filename, git_root, ignore_files):
612
+ continue
613
+ else:
614
+ return False
615
+
616
+ return True
617
+
618
+
619
+ def _hg_is_commit_clean(hg_dir=None, ignore_untracked=True, ignore_files=tuple()):
620
+ """
621
+ Checks if a mercurial directory is clean.
622
+
623
+ By default, a directory is considered clean if all tracked files have no uncommitted changes. Untracked files are
624
+ not considered. Setting ``ignore_untracked`` to ``False`` means that there must be no untracked files for the
625
+ directory to be clean.
626
+
627
+ :param hg_dir: optional, the mercurial directory to check. If not given, defaults to the one containing this repo.
628
+ :type hg_dir: str
629
+
630
+ :param ignore_untracked: optional, set to ``False`` to require that there be no untracked files in the directory for
631
+ it to be considered clean.
632
+ :type ignore_untracked: bool
633
+
634
+ :return: ``True`` if the directory is clean, ``False`` otherwise.
635
+ :rtype: bool
636
+ """
637
+ hg_dir = _vcs_dir_helper(hg_dir)
638
+ hg_root = subprocess.check_output(['hg', 'root'], cwd=hg_dir).strip()
639
+ summary = subprocess.check_output(['hg', 'status'], cwd=hg_dir).splitlines()
640
+
641
+
642
+ # Since subprocess returns a bytes object (at least on Linux) rather than an encoded string object, all the strings
643
+ # below must be bytes, not unicode strings
644
+ for line in summary:
645
+ status, hg_file = [p.strip() for p in line.split(b' ', 1)]
646
+ if ignore_untracked and status == b'?':
647
+ pass
648
+ elif _in_vcs_ignore(hg_file, hg_root, ignore_files):
649
+ pass
650
+ else:
651
+ return False
652
+
653
+ return True
654
+
655
+
656
+ def _lrange(*args):
657
+ # Ensure Python 3 compatibility for adding range() calls together
658
+ r = range(*args)
659
+ if not isinstance(r, list):
660
+ r = list(r)
661
+ return r
662
+
663
+
664
+ def round_to_zero(val):
665
+ sign = np.sign(val)
666
+ return np.floor(np.abs(val)) * sign
667
+
668
+
669
+ def iter_mod_dirs(mod_top_dir, path=None, check_if_exists=False):
670
+ """
671
+ Iterate over available .mod site directories.
672
+
673
+ :param mod_top_dir: the root directory for the .mod files. Must include subdirectories that are the two-letter site
674
+ abbreviations.
675
+ :type mod_top_dir: str
676
+
677
+ :param path: the subpath within each of the site dirs to return. If None, then the site directory itself is
678
+ returned. Otherwise this is appended to the end and that returned.
679
+ :type path: str
680
+
681
+ :param check_if_exists: if ``True`` verify that the subpath exists before yielding. No effect if ``path`` is None.
682
+ :type check_if_exists: bool
683
+
684
+ :return: iterable of .mod file directories.
685
+ :rtype: iterable
686
+ """
687
+ # only match two-character file names
688
+ site_dirs = sorted(glob(os.path.join(mod_top_dir, '??')))
689
+ for sdir in site_dirs:
690
+ if path is None:
691
+ yield sdir
692
+ else:
693
+ fullpath = os.path.join(sdir, path)
694
+ if check_if_exists and not os.path.isdir(fullpath):
695
+ raise ModPathError('Requested subpath "{}" does not exist in {}'.format(path, sdir))
696
+ yield fullpath
697
+
698
+
699
+ def extract_mod_site_abbrevs(mod_files, default='xx'):
700
+ """
701
+ Extract the site abbreviations from .mod file paths
702
+
703
+ The .mod files no longer include the site abbreviation anywhere in the file name or header, so we need to extract
704
+ those from the directory paths. This function assumes paths of the format ".../abbr/<something>/*.mod", i.e. that
705
+ there are are least three levels (including the .mod file) and that the third from the end is the site abbreviation.
706
+
707
+ For each path, if (a) there are not three directory levels, (b) the third part from the end is not a two-letter
708
+ name, or (c) if it is a dictionary, this returns the default abbreviation.
709
+
710
+ :param mod_files: a single path to a .mod file, or a list of .mod files
711
+ :type mod_files: str or list(str)
712
+
713
+ :param default: the default site abbreviation to return if cannot find an abbreviation
714
+ :type default: str
715
+
716
+ :return: a single site abbreviation if ``mod_files`` is a string, a tuple of abbreviations otherwise.
717
+ :rtype: str or tuple(str)
718
+ """
719
+ def extract_single(modf):
720
+ if isinstance(modf, dict):
721
+ return default
722
+ elif not isinstance(modf, str):
723
+ raise TypeError('Expected string or dictionary for each mod file')
724
+
725
+ fileparts = modf.split(os.sep)
726
+ if len(fileparts) < 3:
727
+ return default
728
+ elif not re.match(r'[a-zA-Z0-9]{2}', fileparts[-3]):
729
+ return default
730
+ else:
731
+ return fileparts[-3]
732
+
733
+ if isinstance(mod_files, str):
734
+ return extract_single(mod_files)
735
+ else:
736
+ return tuple([extract_single(modf) for modf in mod_files])
737
+
738
+
739
+ def iter_mod_files(mod_top_dir, include_slant=False):
740
+ """
741
+ Iterate over available .mod files.
742
+
743
+ :param mod_top_dir: the root directory for the .mod files. Must include subdirectories that are the two-letter site
744
+ abbreviations.
745
+ :type mod_top_dir: str
746
+
747
+ :param include_slant: not yet implemented, but in the future will turn on returning pairs of files (vertical and
748
+ slant).
749
+ :type include_slant: bool
750
+
751
+ :return: iterable of paths to .mod files. The paths will start with the given ``mod_top_dir``.
752
+ """
753
+ if include_slant:
754
+ raise NotImplementedError('include_slant not yet implemented')
755
+
756
+ for vdir in iter_mod_dirs(mod_top_dir, 'vertical'):
757
+ vertical_files = sorted(glob(os.path.join(vdir, '*.mod')))
758
+ for f in vertical_files:
759
+ yield f
760
+
761
+
762
+ def calculate_model_potential_temperature(temp, pres_levels=_std_model_pres_levels):
763
+ """
764
+ Calculate potental temperature for model output on fixed pressure levels.
765
+
766
+ :param temp: The absolute temperature (in K) on the model grid.
767
+ :type temp: array-like
768
+
769
+ :param pres_levels: the pressure levels that the temperature is defined on. Must be a 1D vector, i.e. all columns in
770
+ the model must be on the same pressure levels. A standard set of pressure for GEOS FP is the default.
771
+ :type pres_levels: vector-like
772
+
773
+ :return: the potential temperature
774
+ """
775
+ if temp.ndim != 4:
776
+ raise ValueError('temp expected to be 4D')
777
+
778
+ ntime, nlev, nlat, nlon = temp.shape
779
+ if nlev != pres_levels.size:
780
+ raise ValueError('Number of levels in temp != number of pressure levels defined')
781
+ pres = np.tile(pres_levels[np.newaxis, :, np.newaxis, np.newaxis], (ntime, 1, nlat, nlon))
782
+ return calculate_potential_temperature(pres, temp)
783
+
784
+
785
+ def calculate_potential_temperature(pres, temp):
786
+ """
787
+ Calculate potential temperature.
788
+
789
+ :param pres: Pressure in millibars/hPa.
790
+ :param temp: Absolute temperature in Kelvin.
791
+ :return: the potential temperature corresponding to those T/P coordinates.
792
+ """
793
+ return temp * (1000/pres) ** 0.286
794
+
795
+
796
+ def convert_geos_eta_coord(delp):
797
+ """
798
+ Calculate the pressure grid for a GEOS native file.
799
+
800
+ :param pres: DELP (pressure thickness) array in Pa. May be any number of
801
+ dimensions, as long as exactly one has a length of 72.
802
+ :return: the pressure level midpoints, in hPa. Note the unit change, this
803
+ is because the GEOS DELP variable is usually in Pa, but hPa is the standard
804
+ unit for pressure levels in the Np files.
805
+ """
806
+ dp_shape = np.array(delp.shape)
807
+ try:
808
+ i_ax = np.flatnonzero(dp_shape == 72).item()
809
+ except ValueError:
810
+ raise ValueError('delp is either missing its 72 level dimension or has multiple dimensions with length 72')
811
+
812
+ # From pg. 7 of the GEOS FP document (https://gmao.gsfc.nasa.gov/GMAO_products/documents/GEOS_5_FP_File_Specification_ON4v1_2.pdf)
813
+ # the top pressure is always 0.01 hPa. Since the columns are space-to-surface, we add the cumulative sum to get the
814
+ # level bottom pressure, then take the average along that axis to get the middle pressure
815
+ level_shape = dp_shape.copy()
816
+ level_shape[i_ax] = 1
817
+ top_p = 0.01
818
+ top_p_slice = np.full(level_shape, top_p)
819
+
820
+ delp = delp * 0.01 # assume input is in Pa, want hPa
821
+ p_edge = top_p + np.cumsum(delp, axis=i_ax)
822
+ p_edge = np.concatenate([top_p_slice, p_edge], axis=i_ax)
823
+
824
+ # Move the vertical axis to the front to do the averaging so we can just always average along the first dimension
825
+ p_edge = np.rollaxis(p_edge, i_ax, 0)
826
+ p_mid = 0.5 * (p_edge[:-1] + p_edge[1:])
827
+ return np.rollaxis(p_mid, i_ax, 0)
828
+
829
+
830
+ def _construct_grid(*part_defs):
831
+ """
832
+ Helper function to construct coordinates for a 1D grid
833
+ :param part_defs: a sequence of tuples (or lists) defining the start, stop, and step for each grid component. Each
834
+ tuple gets expanded as the arguments to :func:`numpy.arange`.
835
+ :return: the coordinates, sorted, and made sure to be unique
836
+ """
837
+ grid_components = [np.arange(*part) for part in part_defs]
838
+ # Keep only unique values and sort them
839
+ return np.unique(np.concatenate(grid_components))
840
+
841
+
842
+ # Compute area of each grid cell and the total area
843
+ def calculate_area(lat, lon, lat_res=None, lon_res=None, muted=False):
844
+ """
845
+ Calculate grid cell area for an equirectangular grid.
846
+
847
+ :param lat: the vector of grid cell center latitudes, in degrees
848
+ :param lon: the vector of grid cell center longitudes, in degrees
849
+ :param lat_res: the width of a single grid cell in the latitudinal direction, in degrees. If omitted, will be
850
+ calculated from the lat vector.
851
+ :param lon_res: the width of a single grid cell in the longitudinal direction, in degrees. If omitted, will be
852
+ calculated from the lat vector.
853
+ :return: 2D array of areas, in units of fraction of Earth's surface area.
854
+ """
855
+ def calculate_resolution(coord_vec, coord_name):
856
+ res = np.diff(coord_vec)
857
+ if not np.all(res - res[0] < 0.001):
858
+ raise RuntimeError('Could not determine a unique {} resolution'.format(coord_name))
859
+ return res[0]
860
+
861
+ nlat = lat.size
862
+ nlon = lon.size
863
+
864
+ if lat_res is None:
865
+ lat_res = calculate_resolution(lat, 'lat')
866
+ if lon_res is None:
867
+ lon_res = calculate_resolution(lon, 'lon')
868
+
869
+ lat_half_res = 0.5 * lat_res
870
+
871
+ area = np.zeros([nlat, nlon])
872
+
873
+ for j in range(nlat):
874
+ Slat = lat[j]-lat_half_res
875
+ Nlat = lat[j]+lat_half_res
876
+
877
+ Slat = np.deg2rad(Slat)
878
+ Nlat = np.deg2rad(Nlat)
879
+ for i in range(nlon):
880
+ area[j, i] = np.deg2rad(lon_res)*np.abs(np.sin(Slat)-np.sin(Nlat))
881
+
882
+ if abs(np.sum(area) - 4*np.pi) > 0.0001: # ensure proper normalization so the total area of Earth is 4*pi
883
+ #if not muted:
884
+ # print('Total earth area is {:g} not 4pi (difference of {:g}), normalizing to 4pi.'
885
+ # .format(np.sum(area), np.sum(area) - 4*np.pi))
886
+ area *= 4*np.pi/np.sum(area)
887
+
888
+ return area
889
+
890
+
891
+ def calculate_eq_lat_on_grid(EPV, PT, area):
892
+ """
893
+ Calculate equivalent latitude on a 4D grid.
894
+
895
+ :param EPV: the potential vorticity on the 4D grid.
896
+ :type EPV: :class:`numpy.ndarray`
897
+
898
+ :param PT: the potential temperature on the 4D grid.
899
+ :type PT: :class:`numpy.ndarray`
900
+
901
+ :param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
902
+ :type area: :class:`numpy.ndarray`
903
+
904
+ :return: equivalent latitude
905
+ :rtype: :class:`numpy.ndarray`
906
+ """
907
+ EL = np.full_like(PT, np.nan)
908
+
909
+ for itime in range(PT.shape[0]):
910
+ interpolator = calculate_eq_lat(EPV[itime], PT[itime], area)
911
+ # This is probably going to be horrifically slow - but interp2d sometimes gives weird results when called with
912
+ # vectors, so unfortunately we have to call this one element at a time
913
+ pbar = ProgressBar(PT[itime].size, prefix='Calculating eq. lat for time {}/{}:'.format(itime, PT.shape[0]),
914
+ style='counter')
915
+ for i in range(PT[itime].size):
916
+ pbar.print_bar(i)
917
+ ilev, ilat, ilon = np.unravel_index(i, PT[itime].shape)
918
+ this_pt = PT[itime, ilev, ilat, ilon]
919
+ this_epv = EPV[itime, ilev, ilat, ilon]
920
+ EL[itime, ilev, ilat, ilon] = interpolator(this_epv, this_pt)[0]
921
+
922
+ return EL
923
+
924
+
925
+ def calculate_eq_lat(EPV, PT, area):
926
+ """
927
+ Construct an interpolator for equivalent latitude.
928
+
929
+ :param EPV: a 3D grid of potential vorticity
930
+ :type EPV: :class:`numpy.ndarray`
931
+
932
+ :param PT: a 3D grid of potential temperature
933
+ :type PT: :class:`numpy.ndarray`
934
+
935
+ :param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
936
+ :type area: :class:`numpy.ndarray`
937
+
938
+ :return: a 2D interpolator for equivalent latitude, requires potential vorticity and potential temperature as inputs
939
+ :rtype: :class:`scipy.interpolate.interp2d`
940
+
941
+ Note: when querying the interpolator for equivalent latitude, it is often best to call it with scalar values, even
942
+ though that is slower than calling it with the full vector of PV and PT that you wish to get EL for. The problem is
943
+ that scipy 2D interpolators, when given vectors as input, return a grid. This would be fine, except that the values
944
+ corresponding to the vector of PV and PT are not always along the diagonal and so cannot be extracted with
945
+ :func:`numpy.diag`. (I suspect what is happening is that the interpolator sorts the input values when constructing
946
+ the grid, but I have not tested this. -JLL)
947
+ """
948
+ nlev, nlat, nlon = PT.shape
949
+ # Get rid of fill values, this fills the bottom of profiles with the first valid value
950
+ PT[PT > 1e4] = np.nan
951
+ EPV[EPV > 1e8] = np.nan
952
+ for i in range(nlat):
953
+ pd.DataFrame(PT[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
954
+ pd.DataFrame(EPV[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
955
+
956
+ # Define a fixed potential temperature grid, with increasing spacing
957
+ # this is done arbitrarily to get sufficient levels for the interpolation to work well, and not too much for the
958
+ # computations to take less time
959
+ if np.min(PT) > 300 or np.max(PT) < 1000:
960
+ raise ValueError('Potential temperature range is smaller than the [300, 1000] K assumed to create the '
961
+ 'interpolation grid')
962
+
963
+ theta_grid = _construct_grid((round_to_zero(np.nanmin(PT)), 300.0, 2), (300.0, 350.0, 5.0), (350.0, 500.0, 10.0),
964
+ (500.0, 750.0, 20.0), (750.0, 1000.0, 30.0), (1000.0, round_to_zero(np.nanmax(PT)), 100.0))
965
+ new_nlev = np.size(theta_grid)
966
+
967
+ # Get PV on the fixed PT levels ~ 2 seconds per date
968
+ new_EPV = np.zeros([new_nlev, nlat, nlon])
969
+ for i in range(nlat):
970
+ for j in range(nlon):
971
+ new_EPV[:, i, j] = np.interp(theta_grid, PT[:, i, j], EPV[:, i, j])
972
+
973
+ # Compute equivalent latitudes
974
+ EL = np.zeros([new_nlev, 100])
975
+ EPV_thresh = np.zeros([new_nlev, 100])
976
+ for k in range(new_nlev): # loop over potential temperature levels
977
+ maxPV = np.max(new_EPV[k]) # global max PV
978
+ minPV = np.min(new_EPV[k]) # global min PV
979
+
980
+ # define 100 PV values between the min and max PV
981
+ EPV_thresh[k] = np.linspace(minPV,maxPV,100)
982
+
983
+ for l,thresh in enumerate(EPV_thresh[k]):
984
+ area_total = np.sum(area[new_EPV[k]>=thresh])
985
+ x = 1-area_total/(2*np.pi)
986
+
987
+ # With Python 3.10 and those dependencies, I started getting cases where x was *just* outside the -1 to 1 allowed domain,
988
+ # which led to NaNs in the EqL and so bad things downstream. Since values were only slightly outside the domain, it's fine
989
+ # to clip them, but if they go too far outside the expected values, then we may have a bigger problem.
990
+ if x < -1.01 or x > 1.01:
991
+ warn(f'Total area divided by 2*pi (x={x}) is far outside the domain of arcsin in EqL calculation. Clipping to -1 to 1.')
992
+
993
+ if x < -1:
994
+ x = -1
995
+ elif x > 1:
996
+ x = 1
997
+
998
+
999
+ EL[k,l] = np.arcsin(x)*90.0*2/np.pi
1000
+
1001
+ # Define a fixed potential vorticity grid, with increasing spacing away from 0
1002
+ # The last term should ensure that 0 is in the grid
1003
+ pv_grid = _construct_grid((round_to_zero(np.nanmin(EPV_thresh-50.0)), -1000.0, 50.0), (-1000.0, -500.0, 20.0), (-500.0, -100.0, 10.0),
1004
+ (-100.0, -10.0, 1.0), (-10.0, -1.0, 0.1), (-1.0, 1.0, 0.01), (1.0, 10.0, 0.1),
1005
+ (10.0, 100.0, 1.0), (100.0, 500.0, 10.0), (500.0, 1000.0, 20.0),
1006
+ (1000.0, round_to_zero(np.nanmax(EPV_thresh))+50.0, 50.0), (0.0, 0.1))
1007
+
1008
+ # Generate interpolating function to get EL for a given PV and PT
1009
+ interp_EL = np.zeros([new_nlev,len(pv_grid)])
1010
+ for k in range(new_nlev):
1011
+ interp_EL[k] = np.interp(pv_grid,EPV_thresh[k],EL[k])
1012
+
1013
+ return interp2d(pv_grid, theta_grid, interp_EL)
1014
+
1015
+
1016
+ def get_eqlat_profile(interpolator, epv, theta):
1017
+ el = np.full_like(epv, np.nan)
1018
+ for i, (pv, pt) in enumerate(zip(epv, theta)):
1019
+ el[i] = interpolator(pv, pt)
1020
+
1021
+ return el
1022
+
1023
+
1024
+ def calculate_eq_lat_field(EPV, PT, area):
1025
+ """
1026
+ Calculate 3D equivalent latitude from same shape potential vorticity and potiential temperature fields
1027
+
1028
+ :param EPV: a 3D grid of Ertel's potential vorticity
1029
+ :type EPV: :class:`numpy.ndarray`
1030
+
1031
+ :param PT: a 3D grid of potential temperature
1032
+ :type PT: :class:`numpy.ndarray`
1033
+
1034
+ :param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
1035
+ :type area: :class:`numpy.ndarray`
1036
+
1037
+ :param method: the interpolation method used by griddata, one of ['linear','nearest','cubic']
1038
+ :type method: str
1039
+
1040
+ :return: the 3D equivalent latitude
1041
+ :rtype: :class:`numpy.ndarray`
1042
+ """
1043
+ # scipy/dask import here to avoid making it a requirement in setup.py as this is not the main/standard function
1044
+ import dask
1045
+ import dask.array as da
1046
+ from dask.diagnostics import ProgressBar as dask_progress
1047
+ from scipy.interpolate import griddata
1048
+
1049
+ nlev, nlat, nlon = PT.shape
1050
+ # Get rid of fill values, this fills the bottom of profiles with the first valid value
1051
+ PT[PT > 1e4] = np.nan
1052
+ EPV[EPV > 1e8] = np.nan
1053
+ for i in range(nlat):
1054
+ pd.DataFrame(PT[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
1055
+ pd.DataFrame(EPV[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
1056
+
1057
+ # Define a fixed potential temperature grid, with increasing spacing
1058
+ # this is done arbitrarily to get sufficient levels for the interpolation to work well, and not too much for the
1059
+ # computations to take less time
1060
+ if np.min(PT) > 300 or np.max(PT) < 1000:
1061
+ raise ValueError('Potential temperature range is smaller than the [300, 1000] K assumed to create the '
1062
+ 'interpolation grid')
1063
+
1064
+ theta_array = _construct_grid((round_to_zero(np.nanmin(PT)), 300.0, 2), (300.0, 350.0, 5.0), (350.0, 500.0, 10.0),
1065
+ (500.0, 750.0, 20.0), (750.0, 1000.0, 30.0), (1000.0, round_to_zero(np.nanmax(PT)), 100.0))
1066
+ #theta_array = np.geomspace(round_to_zero(np.nanmin(PT)),round_to_zero(np.nanmax(PT)),130)
1067
+ new_nlev = np.size(theta_array)
1068
+
1069
+ # Get PV on the fixed PT levels ~ 2 seconds per date
1070
+ new_EPV = np.zeros([new_nlev, nlat, nlon])
1071
+ for i in range(nlat):
1072
+ for j in range(nlon):
1073
+ new_EPV[:, i, j] = np.interp(theta_array, PT[:, i, j], EPV[:, i, j])
1074
+
1075
+ # Compute equivalent latitudes
1076
+ EL = np.zeros([new_nlev, 100])
1077
+ EPV_thresh = np.zeros([new_nlev, 100])
1078
+ for k in range(new_nlev): # loop over potential temperature levels
1079
+ maxPV = np.max(new_EPV[k]) # global max PV
1080
+ minPV = np.min(new_EPV[k]) # global min PV
1081
+
1082
+ # define 100 PV values between the min and max PV
1083
+ #EPV_thresh[k] = np.unique(np.concatenate([-1.0*np.geomspace(1,np.abs(minPV),50),[0],np.geomspace(1,maxPV,50)]))
1084
+ EPV_thresh[k] = np.linspace(minPV,maxPV,100)
1085
+
1086
+ for l,thresh in enumerate(EPV_thresh[k]):
1087
+ area_total = np.sum(area[new_EPV[k]>=thresh])
1088
+ x = 1-area_total/(2*np.pi)
1089
+
1090
+ # see comment in calculate_eq_lat
1091
+ if x < -1.01 or x > 1.01:
1092
+ warn(f'Total area divided by 2*pi (x={x}) is far outside the domain of arcsin in EqL calculation. Clipping to -1 to 1.')
1093
+
1094
+ if x < -1:
1095
+ x = -1
1096
+ elif x > 1:
1097
+ x = 1
1098
+
1099
+ EL[k,l] = np.arcsin(x)*90.0*2/np.pi
1100
+
1101
+ # pv_array has much less elements than in calculate_eq_lat in order to be used with griddata
1102
+ pv_array = np.unique(np.concatenate([
1103
+ -1*np.geomspace(1e3,-1*round_to_zero(np.nanmin(EPV_thresh)),100),
1104
+ _construct_grid((-1000.0, -500.0, 20.0), (-500.0, -100.0, 10.0),(-100.0, -10.0, 1.0), (-10.0, -1.0, 0.1), (-1.0, 1.0, 0.01), (1.0, 10.0, 0.1),
1105
+ (10.0, 100.0, 1.0), (100.0, 500.0, 10.0), (500.0, 1000.0, 20.0), (0.0,0.1)),
1106
+ np.geomspace(1000,round_to_zero(np.nanmax(EPV_thresh)),100),
1107
+ ]))
1108
+
1109
+ # Interpolate EL (equivalent latitude) on the fixed pv_array and theta_array
1110
+ interp_EL = np.zeros([new_nlev,len(pv_array)])
1111
+ for k in range(new_nlev):
1112
+ interp_EL[k] = np.interp(pv_array,EPV_thresh[k],EL[k])
1113
+
1114
+ pv_grid, theta_grid = np.meshgrid(pv_array,theta_array)
1115
+
1116
+ nonan = ~np.isnan(interp_EL) # there are no nans there, it's just to flatten the arrays
1117
+ flat_pv = pv_grid[nonan]
1118
+ flat_theta = theta_grid[nonan]
1119
+ flat_EL = interp_EL[nonan]
1120
+
1121
+ # Use griddata to compute the equivalent latitude level by level
1122
+ # compute the different levels in parallel with dask
1123
+ final_EL = []
1124
+ for ilev in range(nlev):
1125
+ final_EL.append(dask.delayed(griddata)((flat_pv, flat_theta), flat_EL, (EPV[ilev], PT[ilev])))
1126
+
1127
+ with dask_progress():
1128
+ final_EL = np.stack(dask.compute(*final_EL))
1129
+
1130
+ return final_EL
1131
+
1132
+
1133
+ def _format_geosfp_name(product, file_type, levels, date_time, add_subdir=False):
1134
+ """
1135
+ Create the file name for a GEOS FP or FP-IT file.
1136
+
1137
+ :param product: which GEOS product ('fp' or 'fpit') to use
1138
+
1139
+ :param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
1140
+ :type file_type: str
1141
+
1142
+ :param levels: which levels ('surf', 'p', or 'eta') to use
1143
+ :type levels: str
1144
+
1145
+ :param date_time: the date and time of the desired file. The hour should be a multiple of 3.
1146
+ :type date_time: datetime-like
1147
+
1148
+ :param add_subdir: if ``True``, then the correct subdirectory will be prepended.
1149
+ :type add_subdir: bool
1150
+
1151
+ :return: the file name
1152
+ :rtype: str
1153
+ """
1154
+ if product in {'it'}:
1155
+ return _format_geosit_name(product, file_type, levels, date_time, add_subdir=add_subdir)
1156
+
1157
+ product_patterns = {'fp': 'GEOS.fp.asm.inst3_{dim}d_{vars}_{type}.{date_time}.V01.nc4',
1158
+ 'fpit': 'GEOS.fpit.asm.inst3_{dim}d_{vars}_{type}.GEOS5124.{date_time}.V01.nc4'}
1159
+ level_mapping = {'surf': 'Nx', 'p': 'Np', 'eta': 'Nv'}
1160
+ level_dims = {'Np': 3, 'Nx': 2, 'Nv': 3}
1161
+ var_types = {'met': 'asm', 'chm': 'chm'}
1162
+ try:
1163
+ pattern = product_patterns[product]
1164
+ except KeyError:
1165
+ raise ValueError('product "{}" has not been defined. Allowed values are: {}'
1166
+ .format(product, ', '.join(product_patterns.keys())))
1167
+
1168
+ try:
1169
+ levels = level_mapping[levels]
1170
+ except KeyError:
1171
+ raise ValueError('levels "{}" not recognized. Allowed values are: {}'
1172
+ .format(levels, ', '.join(level_mapping.keys())))
1173
+
1174
+ try:
1175
+ dims = level_dims[levels]
1176
+ except KeyError:
1177
+ raise ValueError('file_type "{}" is not recognized. Allowed values are: {}'
1178
+ .format(file_type, ', '.join(level_dims.keys())))
1179
+
1180
+ date_time = date_time.strftime('%Y%m%d_%H%M')
1181
+ fname = pattern.format(dim=dims, type=levels, date_time=date_time, vars=var_types[file_type])
1182
+ if add_subdir:
1183
+ fname = os.path.join(levels, fname)
1184
+
1185
+ return fname
1186
+
1187
+ def _format_geosit_name(product, file_type, levels, date_time, grid='L', add_subdir=False):
1188
+ product_patterns = {'it': 'GEOS.it.asm.{kind}_inst_{freq}hr_glo_{grid_key}_{lev}.GEOS5294.{dt:%Y-%m-%d}T{dt:%H%M}.V01.nc4'}
1189
+
1190
+ level_descr = {'surf': 'slv', 'eta': 'v72', 'p': 'p42'}
1191
+ level_freq = {'surf': 1, 'eta': 3, 'p': 3}
1192
+ level_subdirs = {'surf': 'Nx', 'eta': 'Nv', 'p': 'Np'}
1193
+ grid_keys = {'L': 'L576x361', 'C': 'C180x180x6'}
1194
+ var_types = {'met': 'asm', 'chm': 'chm'}
1195
+
1196
+ try:
1197
+ pattern = product_patterns[product]
1198
+ except KeyError:
1199
+ raise ValueError('product "{}" has not been defined. Allowed values are: {}'
1200
+ .format(product, ', '.join(product_patterns.keys())))
1201
+
1202
+ try:
1203
+ lev = level_descr[levels]
1204
+ except KeyError:
1205
+ raise ValueError('levels "{}" not recognized. Allowed values are: {}'
1206
+ .format(levels, ', '.join(level_descr.keys())))
1207
+
1208
+ try:
1209
+ freq = level_freq[levels]
1210
+ except KeyError:
1211
+ raise ValueError('level "{}" is not recognized. Allowed values are: {}'
1212
+ .format(levels, ', '.join(level_freq.keys())))
1213
+
1214
+ try:
1215
+ grid_key = grid_keys[grid]
1216
+ except KeyError:
1217
+ raise ValueError('grid "{}" is not recognized. Allowed values are: {}'
1218
+ .format(grid, ', '.join(grid_keys)))
1219
+
1220
+ try:
1221
+ var_type = var_types[file_type]
1222
+ except KeyError:
1223
+ raise ValueError('file_type "{}" is not recognized. Allowed values are: {}'
1224
+ .format(file_type, ', '.join(var_types)))
1225
+
1226
+ fname = pattern.format(kind=var_type, freq=freq, grid_key=grid_key, lev=lev, dt=date_time)
1227
+ if add_subdir:
1228
+ fname = os.path.join(level_subdirs[levels], fname)
1229
+
1230
+ return fname
1231
+
1232
+
1233
+ def read_geos_files(start_date, end_date, geos_path, profile_variables, surface_variables, product='fpit',
1234
+ keep_time_dim=True, concatenate_arrays=False, set_mask_to_nan=False):
1235
+ """
1236
+ Read GEOS FP or FP-IT files between specified dates.
1237
+
1238
+ :param start_date: the first date to read GEOS files from
1239
+ :type start_date: datetime-like
1240
+
1241
+ :param end_date: the last date to read GEOS file from (exclusive). Note that if this datetime is not exactly a time
1242
+ that GEOS files are produced on, you will lose an extra file. For example, since GEOS files are produced every 3
1243
+ hours, if you specify ``end_date = datetime.datetime(2012, 1, 1, 23)``, you will lose the file from
1244
+ 21:00 UTC 1 Jan 2012,
1245
+ :type end_date: datetime-like
1246
+
1247
+ :param geos_path: the path where the GEOS files are stored. Must have subdirectories 'Np' and 'Nx' for the profile
1248
+ and surface files, respectively. Currently, each of these subdirectories must be flat, meaning that all GEOS data
1249
+ from all times is stored at the top level, not organized into further subdirectories by year/month etc.
1250
+ :type geos_path: str
1251
+
1252
+ :param profile_variables: a list of variables to read from the profile (Np) files. 'lon', 'lat', and 'lev' are
1253
+ always read.
1254
+ :type profile_variables: list(str)
1255
+
1256
+ :param surface_variables: a list of variables to read from the surface (Nx) files. 'lon' and 'lat' are always read.
1257
+ :type surface_variables: list(str)
1258
+
1259
+ :param product: one of the strings 'fp' or 'fpit', determine which GEOS product is being read.
1260
+ :type product: str
1261
+
1262
+ :param keep_time_dim: Set to ``True`` to keep the time dimension of the variables. This means the profile and
1263
+ surface variables will be 4D and 3D respectively. Set to ``False`` to remove it (so they will be 3D and 2D).
1264
+ :type keep_time_dim: bool
1265
+
1266
+ :param concatenate_arrays: Set to ``True`` to concatenate the data from different files into a single array. This
1267
+ requires ``keep_time_dim`` to be ``True`` since they are concatenated along the time dimension. If ``False``, then
1268
+ the variables are left as lists of numpy arrays, where each array comes from a separate file.
1269
+ :type concatenate_arrays: bool
1270
+
1271
+ :param set_mask_to_nan: Set to ``True`` turn the masked arrays read from netCDF files by default into regular numpy
1272
+ arrays, where the masked values are replaced with NaNs.
1273
+ :type set_mask_to_nan: bool
1274
+
1275
+ :return: dictionaries of profile and surface variables, and a Pandas DateTimeIndex of the file dates. The variable
1276
+ dictionaries' values' format depends on the value of ``concatenate_arrays``.
1277
+ :rtype: dict, dict, DatetimeIndex
1278
+ """
1279
+ def read_var_helper(nchandle, varname, keep_time=keep_time_dim):
1280
+ if keep_time:
1281
+ data = nchandle.variables[varname][:]
1282
+ else:
1283
+ # This is equivalent to doing nchandle.variables[varname][0,:,:,:] for a 4D variable; omitting the trailing
1284
+ # colons makes it general for any size array.
1285
+ data = nchandle.variables[varname][0]
1286
+
1287
+ if set_mask_to_nan:
1288
+ data = data.filled(np.nan)
1289
+
1290
+ return data
1291
+
1292
+ def read_files_helper(file_list, variables, is_profile):
1293
+ var_data = {v: [] for v in variables}
1294
+ for fidx, fname in enumerate(file_list):
1295
+
1296
+ with ncdf.Dataset(fname, 'r') as nchandle:
1297
+ # Always read lat/lon. If reading a profile file, get the levels too for the vertical coordinate.
1298
+ lon = read_var_helper(nchandle, 'lon', keep_time=True)
1299
+ lat = read_var_helper(nchandle, 'lat', keep_time=True)
1300
+ if is_profile:
1301
+ lev = read_var_helper(nchandle, 'lev', keep_time=True)
1302
+
1303
+ # If on the first file, store the coordinate variables. If on a later file, double check that the
1304
+ # coordinates are the same. They should be, and that assumption makes the data easier to work with
1305
+ # since we don't have to recheck our indices for each file.
1306
+ if fidx == 0:
1307
+ var_data['lon'] = lon
1308
+ var_data['lat'] = lat
1309
+ if is_profile:
1310
+ var_data['lev'] = lev
1311
+ else:
1312
+ chk = not ma.allclose(var_data['lon'], lon) or not ma.allclose(var_data['lat'], lat)
1313
+ if is_profile:
1314
+ chk = chk or not ma.allclose(var_data['lev'], lev)
1315
+
1316
+ if chk:
1317
+ # TODO: replace this with proper GEOS error from the backend analysis
1318
+ raise RuntimeError('lat, lon, and/or lev are inconsistent among the GEOS files')
1319
+
1320
+ for var in variables:
1321
+ var_data[var].append(read_var_helper(nchandle, var))
1322
+
1323
+ if concatenate_arrays:
1324
+ for var, data in var_data.items():
1325
+ if isinstance(data, list):
1326
+ # The lon/lat/lev variables don't need concatenated, we checked that they don't change with time
1327
+ # so there's only one array for them, not a list.
1328
+ var_data[var] = concatenate(data, axis=0)
1329
+
1330
+ return var_data
1331
+
1332
+ # input checking - we concatenate along the time dimension, so we better keep it
1333
+ if concatenate_arrays and not keep_time_dim:
1334
+ raise ValueError('concatenate_arrays = True requires keep_time_dim = True')
1335
+
1336
+ # If we're converting the default masked arrays to regular arrays, we have to use np.concatenate because
1337
+ # ma.concatenate always returns a masked array. If we're not converting, then the reverse applied.
1338
+ if set_mask_to_nan:
1339
+ concatenate = np.concatenate
1340
+ else:
1341
+ concatenate = ma.concatenate
1342
+
1343
+ geos_prof_files, file_dates = geosfp_file_names(product, 'met', 'p', start_date, end_date)
1344
+ geos_surf_files, surf_file_dates = geosfp_file_names(product, 'met', 'surf', start_date, end_date)
1345
+
1346
+ # Check that the file lists have the same dates
1347
+ if len(file_dates) != len(surf_file_dates) or any(file_dates[i] != surf_file_dates[i] for i in range(len(file_dates))):
1348
+ raise RuntimeError('Somehow listed different profile and surface files')
1349
+ elif concatenate_arrays:
1350
+ file_dates = pd.DatetimeIndex(file_dates)
1351
+
1352
+ geos_prof_files = [os.path.join(geos_path, 'Np', f) for f in geos_prof_files]
1353
+ geos_surf_files = [os.path.join(geos_path, 'Nx', f) for f in geos_surf_files]
1354
+ prof_data = read_files_helper(geos_prof_files, profile_variables, is_profile=True)
1355
+ surf_data = read_files_helper(geos_surf_files, surface_variables, is_profile=False)
1356
+ return prof_data, surf_data, file_dates
1357
+
1358
+
1359
+ def geosfp_file_names(product, file_type, levels, start_date, end_date=None):
1360
+ """
1361
+ List all file names for GEOS FP or FP-IT files for the given date(s).
1362
+
1363
+ :param product: which GEOS product ('fp' or 'fpit') to use
1364
+ :type product: str
1365
+
1366
+ :param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
1367
+ :type file_type: str
1368
+
1369
+ :param levels: which levels ('surf', 'p', or 'eta') to use
1370
+ :type levels: str
1371
+
1372
+ :param start_date: what date to start listing files for. If ``end_date`` is omitted, only the file for this date
1373
+ will be listed. Note that the hour must be a multiple of 3, since GEOS files are produced every three hours.
1374
+ :type start_date: datetime-like
1375
+
1376
+ :param end_date: what date to stop list files. This is exclusive, and will not itself be included. Can be omitted
1377
+ to just list one file.
1378
+ :type end_date: None or datetime-like
1379
+
1380
+ :return: the list of file names and an array of file dates
1381
+ :rtype: list, :class:`pandas.DatetimeIndex`
1382
+ """
1383
+ freq = pd.Timedelta(hours=3)
1384
+ if start_date.hour % 3 != 0:
1385
+ raise ValueError('The hour of start_date must be a multiple of 3')
1386
+ if end_date is None:
1387
+ end_date = start_date + freq
1388
+
1389
+ geos_file_dates = pd.date_range(start=start_date, end=end_date - freq, freq=freq)
1390
+ geos_file_names = []
1391
+ for date in geos_file_dates:
1392
+ this_name = _format_geosfp_name(product, file_type, levels, date)
1393
+ geos_file_names.append(this_name)
1394
+
1395
+ return geos_file_names, geos_file_dates
1396
+
1397
+
1398
+ def geosfp_file_names_by_day(product, file_type, levels, utc_dates, utc_hours=None, add_subdir=False):
1399
+ """
1400
+ Create a list of GEOS-FP file names for specified dates
1401
+
1402
+ This differs from :func:`geosfp_file_names` because this function can list files for only specific hours across
1403
+ multiple days. For example, if you want only 00:00 UTC FP-IT profile files for all of 2018, you would call this as::
1404
+
1405
+ geosfp_file_names_by_day('fpit', 'Np', pd.date_range('2018-01-01', '2018-12-31', utc_hours=[0])
1406
+
1407
+ :param product: which GEOS-FP product to make names for: "fp" or "fpit"
1408
+ :type product: str
1409
+
1410
+ :param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
1411
+ :type file_type: str
1412
+
1413
+ :param levels: which levels ('surf', 'p', or 'eta') to use
1414
+ :type levels: str
1415
+
1416
+ :param utc_dates: Dates (on UTC time) to read files for.
1417
+ :type utc_dates: collection(datetime) or collection(datetime-like objects)
1418
+
1419
+ :param utc_hours: Which hours of the day to use (in UTC). If ``None``, then all hours that GEOS is produced on is
1420
+ used (every 3 hours). Otherwise, pass a collection of integers to specify a subset of hours to use. (e.g.
1421
+ [0, 3, 6, 9] to only use the files from the first half of each day).
1422
+ :type utc_hours: None or collection(int)
1423
+
1424
+ :param add_subdir: if ``True``, then the correct subdirectory will be prepended.
1425
+ :type add_subdir: bool
1426
+
1427
+ :return: a list of GEOS file names
1428
+ :rtype: list(str)
1429
+ """
1430
+ geos_utc_hours = np.arange(0, 24, 3)
1431
+ if utc_hours is not None:
1432
+ geos_utc_hours = geos_utc_hours[np.isin(geos_utc_hours, utc_hours)]
1433
+
1434
+ geos_file_names = []
1435
+ geos_file_dates = []
1436
+ for date in utc_dates:
1437
+ for hr in geos_utc_hours:
1438
+ date_time = dt.datetime(date.year, date.month, date.day, hr)
1439
+ this_name = _format_geosfp_name(product, file_type, levels, date_time, add_subdir=add_subdir)
1440
+ geos_file_names.append(this_name)
1441
+ geos_file_dates.append(date_time)
1442
+
1443
+ return geos_file_names, geos_file_dates
1444
+
1445
+
1446
+ def datetime_from_geos_filename(geos_filename):
1447
+ geos_filename = os.path.basename(geos_filename)
1448
+ if geos_filename.startswith('GEOS.it'):
1449
+ date_str = re.search(r'\d{4}-\d{2}-\d{2}T\d{4}', geos_filename).group()
1450
+ return dt.datetime.strptime(date_str, '%Y-%m-%dT%H%M')
1451
+ else:
1452
+ date_str = re.search(r'\d{8}_\d{4}', geos_filename).group()
1453
+ return dt.datetime.strptime(date_str, '%Y%m%d_%H%M')
1454
+
1455
+
1456
+ def is_geos_on_native_grid(geos_filename):
1457
+ with ncdf.Dataset(geos_filename, 'r') as nch:
1458
+ return nch['lev'].size == 72
1459
+
1460
+
1461
+ def mode_to_product(mode):
1462
+ return mode.replace('-eta', '')
1463
+
1464
+
1465
+ def mod_interpolation_legacy(z_grid, z_met, t_met, val_met, interp_mode=1, met_alt_geopotential=True):
1466
+ """
1467
+ Legacy interpolation for .mod file profiles onto the TCCON grid
1468
+
1469
+ :param z_grid: the altitude levels (in kilometers) to interpolate the values onto
1470
+ :type z_grid: :class:`numpy.ndarray`
1471
+
1472
+ :param z_met: the altitude levels (in kilometers) of the input values
1473
+ :type z_met: :class:`numpy.ndarray`
1474
+
1475
+ :param t_met: the absolute temperature (in Kelvin) on the same levels as the input values
1476
+ :type t_met: :class:`numpy.ndarray`
1477
+
1478
+ :param val_met: the input values to be interpolated to the ``z_grid`` levels
1479
+ :type val_met: :class:`numpy.ndarray`
1480
+
1481
+ :param interp_mode: how to do the interpolation. Mode ``1`` (default) is used in the original GGG code for water
1482
+ vapor dry-air mole fraction. Recommended mode for anything relating to a concentration. Mode ``0`` is for
1483
+ temperature only. Mode ``2`` is for pressure, or more generally, values with an exponential dependence on altitude.
1484
+ :type interp_mode: int
1485
+
1486
+ :param met_alt_geopotential: if ``True``, in met altitudes are assumed to be heights based on geopotential, not
1487
+ geometry. Therefore internally, the grid altitudes are slightly modified to be compatible. If ``False``, then the
1488
+ met altitudes are assumed to be geometric, and the grid altitudes will not be scaled.
1489
+ :type met_alt_geopotential: bool
1490
+
1491
+ :return: the input values, interpolated onto the ``z_grid`` altitudes.
1492
+ :rtype: :class:`numpy.ndarray`
1493
+ """
1494
+ if met_alt_geopotential:
1495
+ z_grid = z_grid / (1 + z_grid/earth_radius)
1496
+
1497
+ val_grid = np.full(z_grid.shape, np.nan, dtype=val_met.dtype)
1498
+ for i, z in enumerate(z_grid):
1499
+ # Find the levels in the met data above and below the current grid level. If we're below the first met level,
1500
+ # use the first two levels to extrapolate
1501
+ i_below = np.argwhere(z_met < z)
1502
+ if i_below.size == 0:
1503
+ i_below = 0
1504
+ else:
1505
+ i_below = np.max(i_below)
1506
+
1507
+ i_above = i_below + 1
1508
+
1509
+ # Calculate beta, which is used as the interpolation factor
1510
+ lapse_rate = (t_met[i_above] - t_met[i_below]) / (z_met[i_above] - z_met[i_below])
1511
+ if lapse_rate * (t_met[i_above] - t_met[i_below]) > 0.01 * t_met[i_below]:
1512
+ beta = np.log(1 + lapse_rate * (z - z_grid[i_below])/t_met[i_below]) / np.log(1 + lapse_rate*(z_met[i_below] - z_met[i_below]) / t_met[i_below])
1513
+ else:
1514
+ beta = (z - z_met[i_below]) / (z_met[i_above] - z_met[i_below])
1515
+
1516
+ # Different interpolation modes that come from the GGG2014 fortran code.
1517
+ if interp_mode == 0:
1518
+ # interp_mode = 0 is for temperature only, uses lapse rate directly
1519
+ val_grid[i] = val_met[i_below] + lapse_rate * (z - z_met[i_below])
1520
+ elif interp_mode == 1:
1521
+ # interp_mode = 1 is for species concentrations
1522
+ val_grid[i] = val_met[i_below] + beta * (val_met[i_above] - val_met[i_below])
1523
+ elif interp_mode == 2:
1524
+ # interp_mode = 2 is for pressure
1525
+ val_grid[i] = val_met[i_below] * (val_met[i_above] / val_met[i_below])**beta
1526
+ else:
1527
+ raise ValueError('interp_mode = {} is not allowed. Must be 0, 1, or 2'.format(interp_mode))
1528
+
1529
+ return val_grid
1530
+
1531
+
1532
+ def mod_interpolation_new(z_grid, z_met, vals_met, interp_mode='linear'):
1533
+ """
1534
+ New method to interpolate met data onto the TCCON prior altitude grid
1535
+
1536
+ :param z_grid: the altitude levels to interpolate to.
1537
+ :type z_grid: :class:`numpy.ndarray`
1538
+
1539
+ :param z_met: the altitude levels in the meteorology data
1540
+ :type z_met: :class:`numpy.ndarray`
1541
+
1542
+ :param vals_met: the values to be interpolated, on the ``z_met`` levels
1543
+ :type vals_met: :class:`numpy.ndarray`
1544
+
1545
+ :param interp_mode: how to do the interpolation:
1546
+
1547
+ * ``'linear'`` will do linear interpolation of ``vals_met`` with respect to z.
1548
+ * ``'lin-log'`` linearly interpolate ln(``vals_met``) with respect to z.
1549
+ * ``'log-lin'`` linearly interpolated ``vals_met`` with respect to ln(z).
1550
+ * ``'log-log'`` linearly interpolate ln(``vals_met``) with respect to ln(z).
1551
+
1552
+ For compatibility with `mod_interpolation_legacy`, ``interp_mode`` may also be an integer. ``0`` or ``1`` are
1553
+ aliases for ``'linear'`` and ``2`` is the same as ``'lin-log'``
1554
+ :type interp_mode: str or int
1555
+
1556
+ :return: the values interpolated to the TCCON grid.
1557
+ :rtype: :class:`numpy.ndarray`
1558
+ """
1559
+ interp_mode_compat_mapping = {0: 'linear', 1: 'linear', 2: 'lin-log', 3: 'log-lin', 4: 'log-log'}
1560
+ err_msg = 'interp_mode = {} is invalid. It must be one of the strings "{}" or one of the integers {}'.format(
1561
+ interp_mode, '", "'.join(interp_mode_compat_mapping.values()),
1562
+ ', '.join([str(k) for k in interp_mode_compat_mapping.keys()])
1563
+ )
1564
+ if isinstance(interp_mode, int):
1565
+ try:
1566
+ interp_mode = interp_mode_compat_mapping[interp_mode]
1567
+ except KeyError:
1568
+ raise ValueError(err_msg)
1569
+ elif interp_mode not in interp_mode_compat_mapping.values():
1570
+ raise ValueError(err_msg)
1571
+
1572
+ interp_mode = interp_mode.lower()
1573
+ do_log_x = re.match('^log-\w{3}', interp_mode)
1574
+ do_log_y = re.match('\w{3}-log$', interp_mode)
1575
+
1576
+ if do_log_x:
1577
+ z_grid = np.log(z_grid)
1578
+ z_met = np.log(z_met)
1579
+ if do_log_y:
1580
+ vals_met = np.log(vals_met)
1581
+
1582
+ vals_interp = interp1d(z_met, vals_met, fill_value='extrapolate')
1583
+ vals_grid = vals_interp(z_grid)
1584
+
1585
+ # If doing logarithmic interpolation for the y variable need to restore the output values. (x not returned so no
1586
+ # need to restore.)
1587
+ if do_log_y:
1588
+ vals_grid = np.exp(vals_grid)
1589
+
1590
+ return vals_grid
1591
+
1592
+
1593
+ def interp_tropopause_height_from_pressure(p_trop_met, p_met, z_met):
1594
+ """
1595
+ Calculate the tropopause height by interpolating to the tropopause pressure
1596
+
1597
+ :param p_trop_met: the blended tropopause pressure from GEOS Nx files.
1598
+ :type p_trop_met: float
1599
+
1600
+ :param p_met: the vector of pressure for this profile. Must be in the same units as ``p_trop_met``.
1601
+ :type p_met: array-like
1602
+
1603
+ :param z_met: the vector of altitude levels for this profile.
1604
+ :type z_met: array-like
1605
+
1606
+ :return: the tropopause altitude, in the same units as ``z_met``.
1607
+ :rtype: float
1608
+ """
1609
+ # The age-of-air calculation used for the tropospheric trace gas profile calculation needs the tropopause altitude.
1610
+ # Previously we'd tried finding this by interpolating to the tropopause potential temperature, in order to be
1611
+ # consistent about defining the strat/trop separation by potential temperature. However, potential temperature
1612
+ # does not always behave in a manner that makes interpolating to it straightforward (e.g. it crosses the tropopause
1613
+ # theta 0 or >1 times) so we just use pressure now.
1614
+ z_trop_met = mod_interpolation_new(p_trop_met, p_met, z_met, 'log-lin')
1615
+ if z_trop_met < np.nanmin(z_met):
1616
+ raise RuntimeError('Tropopause altitude calculated to be below the bottom of the profile. Something has '
1617
+ 'gone horribly wrong.')
1618
+ return z_trop_met
1619
+
1620
+
1621
+ def calc_wmo_tropopause(temperature, altitude, limit_to=(5., 18.), raise_error=True):
1622
+ """
1623
+ Find the tropopause altitude using the WMO definition
1624
+
1625
+ The WMO thermal definition of the tropopause is: "the level at which the lapse rate drops to < 2 K/km and the
1626
+ average lapse rate between this and all higher levels within 2 km does not exceed 2 K/km".
1627
+ (quoted in https://www.atmos-chem-phys.net/8/1483/2008/acp-8-1483-2008.pdf, sect. 2.4).
1628
+
1629
+ :param temperature: the temperature profile, in K
1630
+ :type temperature: :class:`numpy.ndarray` (1D)
1631
+
1632
+ :param altitude: the altitude for each level in the temperature profile, in kilometers
1633
+ :type altitude: :class:`numpy.ndarray` (1D)
1634
+
1635
+ :param limit_to: the range of altitudes to limit the search for the tropopause to. This both helps avoid erroneous
1636
+ results and potentially speed up the analysis.
1637
+ :type limit_to: tuple(float, float)
1638
+
1639
+ :param raise_error: If ``True``, this function raises an error if it cannot find the tropopause. If ``False``, it
1640
+ returns a NaN in that case.
1641
+ :type raise_error: bool
1642
+
1643
+ :return: the tropopause altitude in kilometers
1644
+ :rtype: float
1645
+ :raises TropopauseError: if ``raise_error`` is ``True`` and this cannot find the tropopause.
1646
+ """
1647
+
1648
+ # Calculate the lapse rate on the half levels. By definition, a positive lapse rate is a decrease with altitude, so
1649
+ # we need the minus sign.
1650
+ lapse = -np.diff(temperature) / np.diff(altitude)
1651
+ alt_half = altitude[:-1] + np.diff(altitude)/2.0
1652
+
1653
+ # Cut down the data to just the relevant range of altitudes recommended by
1654
+ # https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2003GL018240 (end of sect. 2)
1655
+ zz = (alt_half >= np.min(limit_to)) & (alt_half <= np.max(limit_to))
1656
+ lapse = lapse[zz]
1657
+ alt_half = alt_half[zz]
1658
+
1659
+ # Iterate over the levels. If the lapse rate is < 2 K/km, check that it remains there over the next 2 kilometers
1660
+ for k, (gamma, alt) in enumerate(zip(lapse, alt_half)):
1661
+ if gamma < 2.0:
1662
+ step = 0.1
1663
+ test_alt = np.arange(alt, alt+2.0+step, step)
1664
+ test_lapse = np.interp(test_alt, alt_half, lapse)
1665
+ if np.all(test_lapse < 2.0):
1666
+ # Interpolate to the exact tropopause altitude where the lapse rate first crosses the 2 K/km
1667
+ # np.interp requires the x-coordinates to be sorted, hence the complicated formula to get k_inds
1668
+ k_inds = np.argsort(lapse[[k-1, k]]) + k - 1
1669
+ return np.interp(2.0, lapse[k_inds], alt_half[k_inds])
1670
+
1671
+ # If we get here, we failed to find the tropopause, so return a NaN or raise an error
1672
+ if raise_error:
1673
+ raise TropopauseError('Could not find a level meeting the WMO tropopause condition in the given profile')
1674
+ else:
1675
+ return np.nan
1676
+
1677
+
1678
+ def number_density_air(p, t):
1679
+ """
1680
+ Calculate the ideal dry number density of air in molec. cm^-3
1681
+
1682
+ :param p: pressure in hPa
1683
+ :type p: float or :class:`numpy.ndarray`
1684
+
1685
+ :param t: temperature in K
1686
+ :type t: float or :class:`numpy.ndarray`
1687
+
1688
+ :return: ideal dry number density in molec. cm^-3
1689
+ :rtype: float or :class:`numpy.ndarray`
1690
+ """
1691
+ R = const.gas_const # gas constant in cm^3 * hPa / (mol * K)
1692
+ return p / (R*t) * const.avogadro
1693
+
1694
+
1695
+ def effective_vertical_path(z, zmin, p=None, t=None, nair=None):
1696
+ """
1697
+ Calculate the effective vertical path used by GFIT for a given z/P/T grid.
1698
+
1699
+ :param z: altitudes of the vertical levels. May be any unit, but note that the effective paths will be returned in
1700
+ the same unit.
1701
+ :type z: array-like
1702
+
1703
+ :param zmin: minimum altitude that the light ray reaches. This is given as ``zmin`` in the netCDF files and the .ray
1704
+ files. Must be in the same unit as ``z``.
1705
+ :type zmin: float
1706
+
1707
+ :param p: pressures of the vertical levels. Must be in hPa.
1708
+ :type p: array-like
1709
+
1710
+ :param t: temperatures of the vertical levels. Must be in K.
1711
+ :type t: array-like
1712
+
1713
+ :return: effective vertical paths in the same units as ``z``
1714
+ :rtype: array-like
1715
+ """
1716
+ def integral(dz_in, lrp_in, sign):
1717
+ return dz_in * 0.5 * (1.0 + sign * lrp_in / 3 + lrp_in**2/12 + sign*lrp_in**3/60)
1718
+
1719
+ if nair is not None:
1720
+ d = nair
1721
+ elif p is not None and t is not None:
1722
+ d = number_density_air(p, t)
1723
+ else:
1724
+ raise TypeError('Either nair or p & t must be given')
1725
+
1726
+ vpath = np.zeros_like(d)
1727
+
1728
+ # From gfit/compute_vertical_paths.f, we need to find the first level above zmin
1729
+ # If there is no such level (which should not happen for TCCON), we treat the top
1730
+ # level this way
1731
+ try:
1732
+ klev = np.flatnonzero(z > zmin)[0]
1733
+ except IndexError:
1734
+ klev = np.size(z) - 1
1735
+
1736
+ # from gfit/compute_vertical_paths.f, the calculation for level i is
1737
+ # v_i = 0.5 * dz_{i+1} * (1 - l_{i+1}/3 + l_{i+1}**2/12 - l_{i+1}**3/60)
1738
+ # + 0.5 * dz_i * (1 + l_i/3 + l_i**2/12 + l_i**3/60)
1739
+ # where
1740
+ # dz_i = z_i - z_{i-1}
1741
+ # l_i = ln(d_{i-1}/d_i)
1742
+ # The top level has no i+1 term. This vector addition duplicates that calculation. The zeros padded to the beginning
1743
+ # and end of the difference vectors ensure that when there's no i+1 or i-1 term, it is given a value of 0.
1744
+ dz = np.concatenate([[0.0], np.diff(z[klev:]), [0.0]])
1745
+ log_rp = np.log(d[klev:-1] / d[klev+1:])
1746
+ log_rp = np.concatenate([[0.0], log_rp, [0.0]])
1747
+
1748
+ # The indexing is complicated here, but with how dz and log_rp are constructed, this makes sure that, for vpath[klev],
1749
+ # the first integral(...) term uses dz = z[klev+1] - z[klev] and log_rp = ln(d[klev]/d[klev+1]) and the second integral
1750
+ # term is 0 (as vpath[klev] needs to account for the surface location below). For all other terms, this combines the
1751
+ # contributions from the weight above and below each level, with different integration signs to account for how the
1752
+ # weights increase from the level below to the current level and decrease from the current level to the level above.
1753
+ vpath[klev:] = integral(dz[1:], log_rp[1:], sign=-1) + integral(dz[:-1], log_rp[:-1], sign=1)
1754
+
1755
+ # Now handle the surface - I don't fully understand how this is constructed mathematically, but the idea is that both
1756
+ # the levels in the prior above and below zmin need to contribute to the column, however that contribution needs to be
1757
+ # 0 below zmin.
1758
+
1759
+ dz = z[klev] - z[klev-1]
1760
+ xo = (zmin - z[klev-1])/dz
1761
+ log_rp = 0.0 if d[klev] <= 0 else np.log(d[klev-1]/d[klev])
1762
+ xl = log_rp * (1-xo)
1763
+ vpath[klev-1] += dz * (1-xo) * (1-xo-xl*(1+2*xo)/3 + (xl**2)*(1+3*xo)/12 + (xl**3)*(1+4*xo)/60)/2
1764
+ vpath[klev] += dz * (1-xo) * (1+xo+xl*(1+2*xo)/3 + (xl**2)*(1+3*xo)/12 - (xl**3)*(1+4*xo)/60)/2
1765
+
1766
+ return vpath
1767
+
1768
+
1769
+ def get_ussa_for_alts(alts):
1770
+ """
1771
+ Get temperature and pressure from the US standard atmosphere (USSA) for given altitudes.
1772
+
1773
+ Temperature is interpolated to the requested altitudes linearly, assuming that the lapse rate is constant between
1774
+ the levels defined by the USSA. Pressure is interpolated exponentially, i.e. ln(p) is interpolated linearly.
1775
+
1776
+ :param alts: altitudes, in kilometers, to calculate T and P for.
1777
+ :type alts: float or :class:`numpy.ndarray`
1778
+
1779
+ :return: temperatures (in K) and pressures (in hPa). Arrays will be the same shape as the input altitudes.
1780
+ :rtype: float or :class:`numpy.ndarray`, float or :class:`numpy.ndarray`
1781
+ """
1782
+ # Need to interpolate pressure and temperature to the given altitudes. Will assume that temperature varies linearly
1783
+ # with altitude and pressure varies exponentially. Since p = p0 * exp(-z/H) then ln(p) = ln(p0) - z/H, therefore
1784
+ # we will linearly interpolate ln(p) w.r.t. altitude.
1785
+ z_coord = const.z_ussa
1786
+ t_coord = const.t_ussa
1787
+ p_coord = np.log(const.p_ussa)
1788
+
1789
+ interp_args = {'left': np.nan, 'right': np.nan}
1790
+ t = np.interp(alts, z_coord, t_coord, **interp_args)
1791
+ p = np.exp(np.interp(alts, z_coord, p_coord, **interp_args))
1792
+
1793
+ return t, p
1794
+
1795
+
1796
+ def get_ussa_for_pres(pres):
1797
+ """
1798
+ Get altitude and temperature from the US standard atmosphere (USSA) for given pressures.
1799
+
1800
+ Temperature is interpolated to the requested altitudes linearly, assuming that the lapse rate is constant between
1801
+ the levels defined by the USSA. Pressure is interpolated exponentially, i.e. ln(p) is interpolated linearly.
1802
+
1803
+ :param pres: pressures, in hPa, to calculate z and T for.
1804
+ :type pres: float or :class:`numpy.ndarray`
1805
+
1806
+ :return: temperatures (in K) and altitudes (in km). Arrays will be the same shape as the input altitudes.
1807
+ :rtype: float or :class:`numpy.ndarray`, float or :class:`numpy.ndarray`
1808
+ """
1809
+
1810
+ # Since temperature varies linearly with altitude and altitude varies linearly vs. ln(p), interpolate both by the
1811
+ # log of pressure
1812
+ z_coord = np.flipud(const.z_ussa)
1813
+ t_coord = np.flipud(const.t_ussa)
1814
+ # must flip - np.interp expects its x-coordinates to be increasing.
1815
+ p_coord = np.flipud(np.log(const.p_ussa))
1816
+
1817
+ pres = np.log(pres)
1818
+
1819
+ interp_args = {'left': np.nan, 'right': np.nan}
1820
+ t = np.interp(pres, p_coord, t_coord, **interp_args)
1821
+ z = np.interp(pres, p_coord, z_coord, **interp_args)
1822
+
1823
+ return t, z
1824
+
1825
+
1826
+ def age_of_air(lat, z, ztrop, ref_lat=45.0):
1827
+ """
1828
+ Calculate age of air using a function form from GGG 2014.
1829
+
1830
+ :param lat: the latitude(s) to calculate age of air for
1831
+ :type lat: float or :class:`numpy.ndarray`
1832
+
1833
+ :param z: the altitude(s) to calculate age of air for. If both ``z`` and ``lat`` given as a vectors, they must be
1834
+ the same shape. Must have units of kilometers.
1835
+ :type z: float or :class:`numpy.ndarray`
1836
+
1837
+ :param ztrop: the tropopause altitude, in kilometers.
1838
+ :type ztrop: float
1839
+
1840
+ :param ref_lat: the reference latitude for the cycle. This is where the exponential in latitude is maximized. 45N
1841
+ was chosen as the default as the center of the northern hemisphere, where most anthropogenic emissions are.
1842
+ :type ref_lat: float
1843
+
1844
+ :return: age of air, in years, as a numpy array
1845
+ :rtype: :class:`numpy.ndarray`
1846
+ """
1847
+ # Force z to be a numpy array. This allows us to use numpy indexing for the extra (stratospheric) term below and
1848
+ # simultaneously ensures aoa is always a numpy array.
1849
+ if not isinstance(z, np.ndarray):
1850
+ z = np.array([z])
1851
+
1852
+ fl = lat/22.0
1853
+ aoa = 0.313 - 0.085 * np.exp(-((lat-ref_lat)/18)**2) - 0.268*np.exp(-1.42 * z / (z+ztrop)) * fl / np.sqrt(1+fl**2)
1854
+
1855
+ # We limit the calculation to z > ztrop here because that avoids a divide-by-0 warning
1856
+ # This term is really only kept in for completeness; in practice, it should never be used because we don't use
1857
+ # this term in the stratosphere.
1858
+ extra_term = 7.0 * (z[z > ztrop]-ztrop)/z[z > ztrop]
1859
+ aoa[z > ztrop] += extra_term
1860
+ return aoa
1861
+
1862
+
1863
+ def seasonal_cycle_factor(lat, z, ztrop, fyr, species, ref_lat=45.0):
1864
+ """
1865
+ Calculate a factor to multiply a concentration by to account for the seasonal cycle.
1866
+
1867
+ :param lat: the latitude(s) to calculate age of air for
1868
+ :type lat: float or :class:`numpy.ndarray`
1869
+
1870
+ :param z: the altitude(s) to calculate age of air for. If both ``z`` and ``lat`` given as a vectors, they must be
1871
+ the same shape. Must have units of kilometers.
1872
+ :type z: float or :class:`numpy.ndarray`
1873
+
1874
+ :param ztrop: the tropopause altitude, in kilometers.
1875
+ :type ztrop: float
1876
+
1877
+ :param fyr: the fraction of the year that corresponds to this date. You can convert a date time to this value with
1878
+ :func:`date_to_frac_year`.
1879
+ :type fyr: float
1880
+
1881
+ :param species: a child class of :class:`~tccon_priors.TraceGasTropicsRecord` that defines a gas name and seasonal
1882
+ cycle coefficient. May be an instance of the class or the class itself. If the gas name is "co2", then a
1883
+ CO2-specific parameterization is used.
1884
+ :type species: :class:`~tccon_priors.TraceGasTropicsRecord`
1885
+
1886
+ :param ref_lat: reference latitude for the age of air. Set to 45N as an approximation of where the NH emissions are.
1887
+ :type ref_lat: float
1888
+
1889
+ :return: the seasonal cycle factor as a numpy array. Multiply this by a deseasonalized concentration at (lat, z) to
1890
+ get the concentration including the seasonal cycle
1891
+ """
1892
+ if species.gas_seas_cyc_coeff is None:
1893
+ raise TypeError('The species record ({}) does not define a seasonal cycle coefficient')
1894
+
1895
+ aoa = age_of_air(lat, z, ztrop, ref_lat=ref_lat)
1896
+ if species.gas_name.lower() == 'co2':
1897
+ sv = np.sin(2*np.pi *(fyr - 0.834 - aoa))
1898
+ svnl = sv + 1.80 * np.exp(-((lat - 74)/41)**2)*(0.5 - sv**2)
1899
+ sca = svnl * np.exp(-aoa/0.20)*(1 + 1.33*np.exp(-((lat-76)/48)**2) * (z+6)/(z+1.4))
1900
+ else:
1901
+ sv = np.sin(2*np.pi * (fyr - 0.78)) # basic seasonal variation
1902
+ svl = sv * (lat / 15.0) / np.sqrt(1 + (lat / 15.0)**2.0) # latitude dependence
1903
+ sca = svl * np.exp(-aoa / 0.85) # altitude dependence
1904
+
1905
+ return 1 + sca * species.gas_seas_cyc_coeff
1906
+
1907
+
1908
+ def hf_ch4_slope_fit(yrs, a, b, c, t0):
1909
+ """
1910
+ A fitting function appropriate to fit the trend of CH4 vs. HF slopes
1911
+
1912
+ This function has the form:
1913
+
1914
+ ..math::
1915
+ a * exp(b*(t - t0)) + c
1916
+
1917
+ where t is given in years.
1918
+
1919
+ :param yrs: t in the above equation.
1920
+ :type yrs: :class:`numpy.ndarray`
1921
+
1922
+ :param a, b, c, t0: the fitting parameters in the above equation
1923
+ :type a, b, c, t0: float
1924
+
1925
+ :return: the predicted slopes at ``yrs``
1926
+ """
1927
+ return a * np.exp(b*(yrs - t0)) + c
1928
+
1929
+
1930
+ # from https://stackoverflow.com/a/16562028
1931
+ def isoutlier(data, m=2):
1932
+ d = np.abs(data - np.nanmedian(data))
1933
+ mdev = np.nanmedian(d)
1934
+ s = d / mdev if mdev else np.zeros_like(data)
1935
+
1936
+ return s >= m
1937
+
1938
+
1939
+ def is_tropics(lat, doy, ages):
1940
+ return np.abs(lat) < 20.0
1941
+
1942
+
1943
+ def is_vortex(lat, doy, ages):
1944
+ if not isinstance(doy, np.ndarray):
1945
+ doy = np.full_like(lat, doy)
1946
+ xx_vortex = np.zeros_like(lat, dtype=np.bool_)
1947
+ xx_vortex[(doy > 140) & (doy < 245) & (lat < -55.0) & (ages > 3.25)] = True
1948
+ xx_vortex[(doy > 275) & (doy < 60) & (lat > 55.0) & (ages > 3.25)] = True
1949
+ return xx_vortex
1950
+
1951
+
1952
+ def is_midlat(lat, doy, ages):
1953
+ return ~is_tropics(lat, doy, ages) & ~is_vortex(lat, doy, ages)
1954
+
1955
+
1956
+ def is_overworld(potential_temp, pressure, trop_pres):
1957
+ return (potential_temp >= 380) & (pressure <= trop_pres)
1958
+
1959
+
1960
+ def npdate_to_datetime(numpy_date):
1961
+ numpy_date = numpy_date.astype('datetime64[s]')
1962
+ ts = (numpy_date - np.datetime64('1970-01-01T00:00:00Z', 's')) / np.timedelta64(1, 's')
1963
+ return dt.datetime.utcfromtimestamp(ts)
1964
+
1965
+
1966
+ def date_to_decimal_year(date_in):
1967
+ """
1968
+ Convert a datetime object to a decimal year.
1969
+
1970
+ A decimal year is e.g. 2018.5, where the part before the decimal is the year itself and after the decimal is how
1971
+ much of the year has passed.
1972
+
1973
+ :param date_in: the datetime object to convert
1974
+ :type date_in: datetime-like
1975
+
1976
+ :return: the decimal year
1977
+ :rtype: float
1978
+ """
1979
+ if date_in is not None:
1980
+ return date_in.year + date_to_frac_year(date_in)
1981
+ else:
1982
+ return np.nan
1983
+
1984
+
1985
+ def day_of_year(date_in):
1986
+ return clams_day_of_year(date_in) - 1
1987
+
1988
+
1989
+ def clams_day_of_year(date_in):
1990
+ return float(date_in.strftime('%j'))
1991
+
1992
+
1993
+ def date_to_frac_year(date_in):
1994
+ """
1995
+ Convert a datetime object to a fraction of a year.
1996
+
1997
+ The fraction is essentially how much of the year has passed, so June 1st (of any year) becomes ~0.416.
1998
+
1999
+ Note: this function assumes 365.25 days per year. This is an imperfect solution, ideally a fractional year should
2000
+ really describe the fraction of an orbit the Earth has completed, so only use this for things where +/- a day is an
2001
+ insignificant error.
2002
+
2003
+ :param date_in: the datetime object to convert
2004
+ :type date_in: datetime-like
2005
+
2006
+ :return: the fractional year
2007
+ :rtype: float
2008
+ """
2009
+ return day_of_year(date_in) / days_per_year # since there's about and extra quarter of a day per year that gives us leap years
2010
+
2011
+
2012
+ def frac_year_to_doy(yr_in):
2013
+ """
2014
+ Convert a fractional year to a day of year.
2015
+
2016
+ Internally, this multiplies by 365.25, so see the warning in the docstring for :func:`date_to_frac_year` about
2017
+ its precision. Crucially, this is NOT a reliable inverse operation to :func:`date_to_frac_year`.
2018
+
2019
+ :param yr_in: the fractional year to convert
2020
+ :type yr_in: float
2021
+
2022
+ :return: the number of days since 1 Jan
2023
+ :rtype: float
2024
+ """
2025
+ return yr_in * days_per_year
2026
+
2027
+
2028
+ def frac_years_to_reldelta(frac_year, allow_nans=True):
2029
+ """
2030
+ Convert a fractional year to a :class:`relativedelta` from dateutils.
2031
+
2032
+ Note: like the other fraction/decimal year functions, this assumes 365.25 days/year internally. Therefore, this
2033
+ should function correctly as an inverse operation to date_to_frac_year when added back to Jan 1 of the year in
2034
+ question.
2035
+
2036
+ :param frac_year: the fractional year(s) (e.g 2.5 for 2 and a half years) to convert
2037
+ :type frac_year: float or a collection of floats
2038
+
2039
+ :param allow_nans: whether to permit NaNs in the decimal years. If ``True``, then NaNs will be retained in the
2040
+ output list. If ``False``, an error is raised if NaNs are found in ``dec_year``.
2041
+ :type allow_nans: bool
2042
+
2043
+ :return: a list of dateutils :class:`relativedelta` objects or a single :class:`relativedelta` if a scalar
2044
+ ``frac_year`` was given.
2045
+ :rtype: :class:`relativedelta` or list(:class:`relativedelta`)
2046
+ """
2047
+ if isinstance(frac_year, float):
2048
+ return_scalar = True
2049
+ frac_year = [frac_year]
2050
+ else:
2051
+ return_scalar = False
2052
+ if not allow_nans and np.any(np.isnan(frac_year)):
2053
+ raise ValueError('NaNs not permitted in frac_year. Either remove them, or set `allow_nans=True`')
2054
+ age_years = np.floor(frac_year)
2055
+ age_fracs = np.mod(frac_year, 1)
2056
+ rdels = [relativedelta(years=y, days=days_per_year * d) if not (np.isnan(y) or np.isnan(d)) else np.nan for y, d in zip(age_years, age_fracs)]
2057
+ if return_scalar:
2058
+ rdels = rdels[0]
2059
+
2060
+ return rdels
2061
+
2062
+
2063
+ def timedelta_to_frac_year(timedelta):
2064
+ """
2065
+ Convert a concrete timedelta to fractional years
2066
+
2067
+ :param timedelta: the timedelta to convert
2068
+ :type timedelta: :class:`datetime.timedelta`
2069
+
2070
+ :return: the time delta as a fraction of years, assuming 365.25 days per year.
2071
+ :rtype: float
2072
+ """
2073
+
2074
+ return timedelta.total_seconds() / (days_per_year * 24 * 3600)
2075
+
2076
+
2077
+ def decimal_year_to_date(dec_year, date_type=dt.datetime):
2078
+ """
2079
+ Convert decimal year or years (e.g. 2018.5) to a datetime-like object
2080
+
2081
+ :param dec_year: the decimal year or years to convert. May be any kind of collection (if passing multiple values) so
2082
+ long as it supports iteration and that iteration returns scalar values (i.e. a 2D numpy array will not work because
2083
+ iteration returns rows).
2084
+
2085
+ :param date_type: what type to convert the decimal years in to. May be any time that can be called
2086
+ ``date_type(year, month, day)``.
2087
+
2088
+ :return: the converted dates in the type ``date_type``. If a single decimal date was passed in, then a single value
2089
+ of type ``date_type`` is returned. If a collection was passed in, then the dates will be returned in a 1D numpy
2090
+ array.
2091
+ """
2092
+
2093
+ if np.any(np.isnan(dec_year)):
2094
+ raise NotImplementedError('NaNs in the input decimal years not implemented')
2095
+
2096
+ try:
2097
+ dec_year[0]
2098
+ except (TypeError, IndexError):
2099
+ dec_year = [dec_year]
2100
+ return_as_scalar = True
2101
+ else:
2102
+ return_as_scalar = False
2103
+
2104
+ years = np.array([int(d) for d in dec_year])
2105
+ frac_yrs = np.array([d % 1 for d in dec_year])
2106
+ dates = np.array([date_type(y, 1, 1) + frac_years_to_reldelta(fr, allow_nans=False) for y, fr in zip(years, frac_yrs)])
2107
+
2108
+ if return_as_scalar:
2109
+ return dates[0]
2110
+ else:
2111
+ return dates
2112
+
2113
+
2114
+ def start_of_month(date_in, out_type=dt.date):
2115
+ """
2116
+ Get a date-like object corresponding to the beginning of the month of ``date_in``
2117
+
2118
+ :param date_in: Any date-like object that has attributes ``year`` and ``month``
2119
+ :param out_type: A type whose constructor accepts the keyword arguments ``year``, ``month``, and ``day``.
2120
+ :return: an instance of ``out_type`` set to day 1, 00:00:00 of the month of ``date_in``.
2121
+ """
2122
+ return out_type(year=date_in.year, month=date_in.month, day=1)
2123
+
2124
+
2125
+ def relativedelta2string(rdelta):
2126
+ parts = ('years', 'months', 'days', 'hours', 'minutes', 'seconds')
2127
+ time_parts = []
2128
+ for p in parts:
2129
+ tp = getattr(rdelta, p)
2130
+ if tp > 0:
2131
+ time_parts.append('{} {}'.format(tp, p))
2132
+
2133
+ return ', '.join(time_parts)
2134
+
2135
+
2136
+ def gravity(gdlat,altit):
2137
+ """
2138
+ copy/pasted from fortran routine comments
2139
+ This is used to convert
2140
+
2141
+ Input Parameters:
2142
+ gdlat GeoDetric Latitude (degrees)
2143
+ altit Geometric Altitude (km)
2144
+
2145
+ Output Parameter:
2146
+ gravity Effective Gravitational Acceleration (m/s2)
2147
+ radius Radius of earth at gdlat
2148
+
2149
+ Computes the effective Earth gravity at a given latitude and altitude.
2150
+ This is the sum of the gravitational and centripital accelerations.
2151
+ These are based on equation I.2.4-(17) in US Standard Atmosphere 1962
2152
+ The Earth is assumed to be an oblate ellipsoid, with a ratio of the
2153
+ major to minor axes = sqrt(1+con) where con=.006738
2154
+ This eccentricity makes the Earth's gravititational field smaller at
2155
+ the poles and larger at the equator than if the Earth were a sphere
2156
+ of the same mass. [At the equator, more of the mass is directly
2157
+ below, whereas at the poles more is off to the sides). This effect
2158
+ also makes the local mid-latitude gravity field not point towards
2159
+ the center of mass.
2160
+
2161
+ The equation used in this subroutine agrees with the International
2162
+ Gravitational Formula of 1967 (Helmert's equation) within 0.005%.
2163
+
2164
+ Interestingly, since the centripital effect of the Earth's rotation
2165
+ (-ve at equator, 0 at poles) has almost the opposite shape to the
2166
+ second order gravitational field (+ve at equator, -ve at poles),
2167
+ their sum is almost constant so that the surface gravity could be
2168
+ approximated (.07%) by the simple expression g=0.99746*GM/radius^2,
2169
+ the latitude variation coming entirely from the variation of surface
2170
+ r with latitude. This simple equation is not used in this subroutine.
2171
+ """
2172
+
2173
+ d2r=3.14159265/180.0 # Conversion from degrees to radians
2174
+ gm=3.9862216e+14 # Gravitational constant times Earth's Mass (m3/s2)
2175
+ omega=7.292116E-05 # Earth's angular rotational velocity (radians/s)
2176
+ con=0.006738 # (a/b)**2-1 where a & b are equatorial & polar radii
2177
+ shc=1.6235e-03 # 2nd harmonic coefficient of Earth's gravity field
2178
+ eqrad=6378178.0 # Equatorial Radius (meters)
2179
+
2180
+ gclat=arctan(tan(d2r*gdlat)/(1.0+con)) # radians
2181
+
2182
+ radius=1000.0*altit+eqrad/np.sqrt(1.0+con*sin(gclat)**2)
2183
+ ff=(radius/eqrad)**2
2184
+ hh=radius*omega**2
2185
+ ge=gm/eqrad**2 # = gravity at Re
2186
+
2187
+ gravity=(ge*(1-shc*(3.0*sin(gclat)**2-1)/ff)/ff-hh*cos(gclat)**2)*(1+0.5*(sin(gclat)*cos(gclat)*(hh/ge+2.0*shc/ff**2))**2)
2188
+
2189
+ return gravity, radius
2190
+
2191
+
2192
+ def geopotential_height_to_altitude(gph, lat, alt):
2193
+ """
2194
+ Convert a geopotential height in m^2 s^-2 to meters
2195
+
2196
+ :param gph: geopotential height in m^2 s^-2.
2197
+ :type gph: float
2198
+
2199
+ :param lat: geographic latitude (in degrees, south is negative)
2200
+ :type lat: float
2201
+
2202
+ :param alt: altitude of the TCCON site in kilometers. If set to 0, will use gravity at the surface for the given
2203
+ latitude.
2204
+ :type alt: float
2205
+
2206
+ :return: the geopotential height converted to meters
2207
+ """
2208
+ gravity_at_site, _ = gravity(lat, alt)
2209
+ return gph / gravity_at_site
2210
+
2211
+
2212
+ def to_unix_time(datetime):
2213
+ """
2214
+ Convert a datetime-like object into Unix time (seconds since midnight, 1 Jan 1970)
2215
+
2216
+ :param datetime: the datetime to convert. May be any type that can have a :class:`datetime.datetime` object
2217
+ subtracted from it, and for which the subtraction has a method `total_seconds` that returns the time delta as
2218
+ a number of seconds. Both :class:`datetime.datetime` and :class:`pandas.Timestamp` are examples.
2219
+
2220
+ :return: unix time
2221
+ :rtype: float
2222
+ """
2223
+ return (datetime - dt.datetime(1970, 1, 1)).total_seconds()
2224
+
2225
+
2226
+ def from_unix_time(utime, out_type=dt.datetime):
2227
+ """
2228
+ Convert a unix time into a datetime object.
2229
+
2230
+ :param utime: the unix time (seconds since midnight, 1 Jan 1970)
2231
+ :type utime: float
2232
+
2233
+ :param out_type: optional, a type that represents a datetime which has an init method such that
2234
+ ``out_type(year, month, day)`` returns a object representing that time and can be added with a
2235
+ :class:`datetime.timedelta``.
2236
+ :type out_type: type
2237
+
2238
+ :return: a datetime object of the type specified by ``out_type``.
2239
+ """
2240
+ return out_type(1970, 1, 1) + dt.timedelta(seconds=utime)
2241
+
2242
+
2243
+ def mod_file_name(prefix,date,time_step,site_lat,site_lon_180,ew,ns,mod_path,round_latlon=True,in_utc=True):
2244
+
2245
+ YYYYMMDD = date.strftime('%Y%m%d')
2246
+ HH = date.strftime('%H')
2247
+ if in_utc:
2248
+ HH += 'Z'
2249
+ else:
2250
+ HH += 'L'
2251
+
2252
+ if round_latlon:
2253
+ site_lat = round(abs(site_lat))
2254
+ site_lon = round(abs(site_lon_180))
2255
+ latlon_precision = 0
2256
+ else:
2257
+ site_lat = abs(site_lat)
2258
+ site_lon = abs(site_lon_180)
2259
+ latlon_precision = 2
2260
+ if time_step < timedelta(days=1):
2261
+ mod_fmt = '{{prefix}}_{{ymd}}{{hh}}_{{lat:0>2.{prec}f}}{{ns:>1}}_{{lon:0>3.{prec}f}}{{ew:>1}}.mod'.format(prec=latlon_precision)
2262
+ else:
2263
+ mod_fmt = '{{prefix}}_{{ymd}}_{{lat:0>2.{prec}f}}{{ns:>1}}_{{lon:0>3.{prec}f}}{{ew:>1}}.mod'.format(prec=latlon_precision)
2264
+
2265
+ mod_name = mod_fmt.format(prefix=prefix, ymd=YYYYMMDD, hh=HH, lat=site_lat, ns=ns, lon=site_lon, ew=ew)
2266
+ return mod_name
2267
+
2268
+
2269
+ def mod_file_name_for_priors(datetime, site_lat, site_lon_180, prefix='FPIT', **kwargs):
2270
+ if site_lon_180 > 180:
2271
+ site_lon_180 -= 360
2272
+ ew = format_lon(site_lon_180)[-1]
2273
+ ns = format_lat(site_lat)[-1]
2274
+ return mod_file_name(prefix=prefix, date=datetime, time_step=dt.timedelta(hours=3), site_lat=site_lat,
2275
+ site_lon_180=site_lon_180, ew=ew, ns=ns, mod_path='', **kwargs)
2276
+
2277
+
2278
+ def parse_date_range(datestr):
2279
+ def parse_date(datestr):
2280
+ try:
2281
+ date_out = dt.datetime.strptime(datestr, '%Y%m%d')
2282
+ except ValueError:
2283
+ date_out = dt.datetime.strptime(datestr, '%Y%m%d_%H')
2284
+ return date_out
2285
+
2286
+ dates = datestr.split('-')
2287
+ start_date = parse_date(dates[0])
2288
+ if len(dates) > 1:
2289
+ end_date = parse_date(dates[1])
2290
+ else:
2291
+ end_date = start_date + timedelta(days=1)
2292
+
2293
+ return start_date, end_date
2294
+
2295
+
2296
+ def get_ggg_path(subdir, subdir_name):
2297
+ gggpath = os.getenv('GGGPATH')
2298
+ if gggpath is None:
2299
+ raise GGGPathError('Could not find the GGGPATH environmental variable. Please specify an explicit {}.'.format(subdir_name))
2300
+ full_subdir = os.path.join(gggpath, subdir)
2301
+ if not os.path.isdir(full_subdir):
2302
+ raise GGGPathError('Could not find default {} {}'.format(subdir_name, full_subdir))
2303
+
2304
+ return full_subdir
2305
+
2306
+
2307
+ def check_site_lat_lon_alt(abbrev, lat=None, lon=None, alt=None):
2308
+ """
2309
+ Verify that the input site abbreviation, latitude, longitude, and altitude are consistent.
2310
+
2311
+ This checks that (1) if any of lat/lon/alt are given, all are, (2) if any inputs are > 1 long all those that are
2312
+ are the same length, (3) if any inputs are scalar (None, int, float, or string) they are converted to tuples
2313
+ of the proper length, and (4) all longitudes are given in [0, 360)
2314
+
2315
+ :param abbrev: site abbreviation(s), as string or iterable of such
2316
+ :type abbrev: str or list(str)
2317
+
2318
+ :param lat: site latitude(s), as float or iterable of such, or None
2319
+ :type lat: float or list(float)
2320
+
2321
+ :param lon: site longitude(s), as float or iterable of such, or None
2322
+ :type lon: float or list(float)
2323
+
2324
+ :param alt: site altitude(s), as float or iterable of such, or None
2325
+ :type alt: float or list(float)
2326
+
2327
+ :return: abbrev, lat, lon, and "alt" converted to tuples of the same length. Scalar values are repeated.
2328
+ :rtype: tuples
2329
+ """
2330
+ def to_tuple(val):
2331
+ if isinstance(val, (int, float, str, type(None))):
2332
+ val = (val,)
2333
+ return val, len(val)
2334
+
2335
+ are_none = [x is None for x in (lat, lon, alt)]
2336
+ abb_tuple, _ = to_tuple(abbrev)
2337
+ if all(are_none):
2338
+ if len(abb_tuple) == 1 and abb_tuple[0] == 'all':
2339
+ pass
2340
+ elif any(abb not in site_dict for abb in abb_tuple):
2341
+ raise ValueError('One or more site abbreviation not a predefined TCCON site, must give lat, lon, alt')
2342
+ elif any(are_none):
2343
+ if len(abb_tuple) == 1 and abb_tuple[0] == 'all':
2344
+ raise ValueError('Running all sites is incompatible with giving custom lat/lons')
2345
+ else:
2346
+ raise ValueError('If any of lat, lon, or alt are given, all must be given')
2347
+
2348
+ vals = dict()
2349
+ n = dict()
2350
+ vals['abbrev'], n['abbrev'] = to_tuple(abbrev)
2351
+ vals['lat'], n['lat'] = to_tuple(lat)
2352
+ vals['lon'], n['lon'] = to_tuple(lon)
2353
+ vals['alt'], n['alt'] = to_tuple(alt)
2354
+
2355
+ max_n = max(v for v in n.values())
2356
+ for k in vals.keys():
2357
+ if n[k] == 1:
2358
+ vals[k] *= max_n
2359
+ elif n[k] != max_n:
2360
+ raise ValueError('{} is not a single value or the same length as the longest input'.format(k))
2361
+
2362
+ # Constrain longitude between 0 and 360
2363
+ vals['lon'] = tuple([l + 360 if l is not None and l < 0 else l for l in vals['lon']])
2364
+ if any([l < 0 or l >= 360 for l in vals['lon'] if l is not None]):
2365
+ raise ValueError('Longitude outside of valid range (-360, +360, exclusive) found')
2366
+
2367
+ return vals['abbrev'], vals['lat'], vals['lon'], vals['alt']
2368
+
2369
+
2370
+ def dry2wet(dmf, h2o_dmf):
2371
+ return dmf / (1 + h2o_dmf)
2372
+
2373
+
2374
+ def interp_to_zgrid(profile_dict, zgrid, gas_extrap_method='linear'):
2375
+ """
2376
+ Interpolate the output profile variables to a desired altitude grid.
2377
+
2378
+ This mimics the behavior in `read_refvmrs` of the Fortran gsetup code. It interpolates the output variables linearly
2379
+ with respect to altitude. It will extrapolate linearly towards the surface, but will raise an error if the profile
2380
+ does not go high enough.
2381
+
2382
+ :param profile_dict: a dictionary of profile variables. Must include the altitude levels as the key "Height". The
2383
+ keys "gas_record_dates" and "gas_date" are treated as dates for interpolation.
2384
+ :type profile_dict: dict
2385
+
2386
+ :param zgrid: any specification of a fixed altitude grid accepted by :func:`_setup_zgrid`, i.e. ``None`` to do no
2387
+ interpolation, a path to an integral file, or an array directly specifying the altitude grid.
2388
+ :type zgrid: None, str, or :class:`numpy.ndarray`.
2389
+
2390
+ :return: the profile dictionary with its variables interpolated/extrapolated to the new altitude grid. It will
2391
+ also have been modified in-place.
2392
+ :rtype: dict
2393
+ """
2394
+ # Specify pairs of functions to convert and unconvert the values of the input arrays. Each key must match a key in
2395
+ # the profile_dict, and the values must be two element tuples, where the first element is the function to use to
2396
+ # convert the values in the dict to ones that can be interpolated and the second does the reverse process.
2397
+ met_variables = ('Height', 'Temp', 'Pressure', 'PT', 'EqL')
2398
+ dt_converters = (_datetime2float, _float2datetime)
2399
+ converters = {'gas_record_dates': dt_converters,
2400
+ 'gas_date': dt_converters}
2401
+
2402
+ zgrid = _setup_zgrid(zgrid)
2403
+ if zgrid is None:
2404
+ return profile_dict
2405
+
2406
+ profile_z = profile_dict['Height'].copy()
2407
+ for k, v in profile_dict.items():
2408
+ if k in converters:
2409
+ v = converters[k][0](v)
2410
+
2411
+ v = xr.DataArray(v, coords=[profile_z], dims=['z'])
2412
+ if gas_extrap_method == 'linear' or k in met_variables:
2413
+ # we always want to linearly extrapolate the met variables, especially since they often act as vertical
2414
+ # coordinates. If more met variables are added, you'll have to update the list.
2415
+ fills = 'extrapolate'
2416
+ elif gas_extrap_method in ('const', 'constant'):
2417
+ fills = (v[0], v[-1])
2418
+ else:
2419
+ raise ValueError('extrap_method must be one of: "linear", "const", or "constant".')
2420
+ v = v.interp(z=zgrid, method='linear', kwargs={'fill_value': fills})
2421
+
2422
+ if k in converters:
2423
+ v = converters[k][1](v.data)
2424
+ else:
2425
+ v = v.data
2426
+ profile_dict[k] = v
2427
+
2428
+ return profile_dict
2429
+
2430
+
2431
+ def _setup_zgrid(zgrid):
2432
+ """
2433
+ Setup a fixed altitude grid
2434
+ :param zgrid:
2435
+ :return:
2436
+ """
2437
+ if isinstance(zgrid, str):
2438
+ int_info = _read_integral_file(zgrid)
2439
+ return int_info['Height']
2440
+ elif not isinstance(zgrid, (type(None), np.ndarray, xr.DataArray)):
2441
+ raise TypeError('zgrid must be a string, None, numpy array, or xarray DataArray')
2442
+ else:
2443
+ return zgrid
2444
+
2445
+
2446
+ def _read_integral_file(integral_file, as_dataframes=False):
2447
+ """
2448
+ Read an integral file that defines an altitude grid for GGG
2449
+
2450
+ :param integral_file: the path to the integral file
2451
+ :type integral_file: str
2452
+
2453
+ :param as_dataframes: if ``True``, the information in the file is returned as a single dataframe. If ``False``, it
2454
+ is returned as a dict of numpy arrays.
2455
+ :type as_dataframes: bool
2456
+
2457
+ :return: the table of altitudes and mean molecular weights.
2458
+ :rtype: :class:`pandas.DataFrame` or dict
2459
+ """
2460
+ df = pd.read_csv(integral_file, sep=r'\s+', header=None, names=['Height', 'mmw'])
2461
+ if as_dataframes:
2462
+ return df
2463
+ else:
2464
+ return {k: v.to_numpy() for k, v in df.items()}
2465
+
2466
+
2467
+ def _datetime2float(dtarray):
2468
+ return np.array([np.nan if d is None else to_unix_time(d) for d in dtarray])
2469
+
2470
+
2471
+ def _float2datetime(dtarray):
2472
+ return np.array([None if np.isnan(d) else from_unix_time(d, pd.Timestamp) for d in dtarray])
2473
+
2474
+
2475
+ def frac_ydh_to_date(year, day, hour):
2476
+ """
2477
+ Convert a fractional year, day, and hour from a GGG output file to a date
2478
+
2479
+ Some GGG output files record year as :math:`year + (day + hour/24)/366`
2480
+ and day as :math:`day + hour/24`. This function converts such values to
2481
+ a Pandas DatetimeIndex.
2482
+
2483
+ :param year: array of years, as decimal numbers
2484
+ :type year: array(float)
2485
+
2486
+ :param day: array of days-of-year, with Jan 1 = 1, as decimal numbers.
2487
+ :type day: array(float)
2488
+
2489
+ :param hour: array of hours of day.
2490
+ :type hour: array(float)
2491
+
2492
+ :return: array of datetimes
2493
+ :rtype: :class:`pandas.DatetimeIndex`
2494
+ """
2495
+ # In many output files, the year and day are calculated to represent the
2496
+ # whole date, i.e. year is a decimal number where the decimal part is the
2497
+ # day + hour in fraction of a year. Same for day. Normally we could just
2498
+ # convert them to an integer and be fine, except there are cases where the
2499
+ # day rolls over into the next day, e.g. because we're working on a western
2500
+ # hemisphere site that starts measuring in one UTC day and keeps measuring
2501
+ # through to the next, so the hour exceeds 24.
2502
+ #
2503
+ # We account for this by subtracting off the fractional part of the year and
2504
+ # day computed from the smaller elements. However, we don't subtract the
2505
+ # whole amount off, because we don't want floating point uncertainty to
2506
+ # take us to less than the correct integer. E.g. if the day is 60.5 and
2507
+ # we compute hour/24 = 0.50000000001, then int(day - hour/24) = 59.
2508
+
2509
+ # correct the year
2510
+ year_frac = day / 366
2511
+ year = (year - 0.99*year_frac).astype('int')
2512
+
2513
+ # correct the day
2514
+ day_frac = hour / 24
2515
+ day = (day - 0.99*day_frac).astype('int')
2516
+
2517
+ return ydh_to_date(year, day, hour)
2518
+
2519
+
2520
+ def ydh_to_date(year, day, hour):
2521
+ dates = []
2522
+ for y, d, h in zip(year, day, hour):
2523
+ ts = pd.Timestamp(y, 1, 1) + pd.Timedelta(days=d-1, hours=h)
2524
+ dates.append(ts)
2525
+ return pd.DatetimeIndex(dates)
2526
+
2527
+
2528
+ def _split_outside_group(s, splitchar, open='(', close=')'):
2529
+ out = []
2530
+ grp_cnt = 0
2531
+ start = 0
2532
+ for i, c in enumerate(s):
2533
+ if c == splitchar and grp_cnt == 0:
2534
+ out.append(s[start:i+1])
2535
+ start = i+1
2536
+ elif c == open:
2537
+ grp_cnt += 1
2538
+ elif c == close:
2539
+ grp_cnt -= 1
2540
+
2541
+ if grp_cnt < 0:
2542
+ raise ValueError('Given string has an unmatched {} at position {}'.format(close, i))
2543
+
2544
+ # Group count should be 0. If not, then there were more open characters than closing characters
2545
+ if grp_cnt > 0:
2546
+ raise ValueError('Given string had an unmatched {}'.format(open))
2547
+
2548
+ # Handle the last piece. Check that we haven't just added something, i.e. that the split character isn't the last
2549
+ # one in the string. If it is, then we should add an empty element
2550
+ if start == len(s) + 1:
2551
+ out.append('')
2552
+ else:
2553
+ out.append(s[start:])
2554
+
2555
+ return out
2556
+
2557
+
2558
+ def fortran_fmt_to_fwf_tuples(fmt_str):
2559
+ fmt_str = fmt_str.strip()
2560
+ if not fmt_str.startswith('(') or not fmt_str.endswith(')'):
2561
+ raise ValueError('The first and last non-whitespace characters of the format string must be open and close '
2562
+ 'parentheses, respectively.')
2563
+ fmt_parts = _split_outside_group(fmt_str[1:-1], ',')
2564
+ colspecs = []
2565
+ types = []
2566
+
2567
+ idx = 0
2568
+ for part in fmt_parts:
2569
+ if '(' in part:
2570
+ match = re.match(r'(\d*)(\(.+\))', part)
2571
+ repeats = 1 if len(match.group(1)) == 0 else int(match.group(1))
2572
+ subfmt = match.group(2)
2573
+ subspecs, subtypes = fortran_fmt_to_fwf_tuples(subfmt)
2574
+
2575
+ for i in range(repeats):
2576
+ for start, stop in subspecs:
2577
+ colspecs.append((idx+start, idx+stop))
2578
+ types += subtypes
2579
+ idx += stop
2580
+ elif re.match(r'\d+x', part):
2581
+ # In Fortran, specifiers like "1x" mean that n spaces do not contain information and are just spacers
2582
+ nspaces = int(re.match(r'\d+', part).group(), re.IGNORECASE)
2583
+ idx += nspaces
2584
+ else:
2585
+ match = re.match(r'(\d*)([a-z])(\d+)', part, re.IGNORECASE)
2586
+ repeats = 1 if len(match.group(1)) == 0 else int(match.group(1))
2587
+ width = int(match.group(3))
2588
+
2589
+ for i in range(repeats):
2590
+ colspecs.append((idx, idx+width))
2591
+ types.append(match.group(2))
2592
+ idx += width
2593
+
2594
+ return colspecs, types
2595
+
2596
+
2597
+ def make_geos_date_ranges(dates):
2598
+ geos_tdel = pd.Timedelta(hours=3)
2599
+ geos_dates = dates.round('3H').unique().sort_values()
2600
+ date_ranges = []
2601
+
2602
+ range_start = geos_dates[0]
2603
+ last_date = geos_dates[0]
2604
+
2605
+ for d in geos_dates[1:]:
2606
+ if d - last_date > geos_tdel:
2607
+ # If the next time is more than the GEOS time step in the future, then the current date range is done.
2608
+ # Append it, remembering that the end date is exclusive
2609
+ date_ranges.append((range_start, last_date + geos_tdel))
2610
+ range_start = d
2611
+ last_date = d
2612
+
2613
+ # There will always be a last date range, even if we added a date range in the last loop (in that case only the last
2614
+ # time will go in the final date range)
2615
+ date_ranges.append((range_start, last_date+geos_tdel))
2616
+ return date_ranges
2617
+
2618
+
2619
+ def get_runlog_geos_date_ranges(rldf):
2620
+ rl_dates = ydh_to_date(rldf['Year'], rldf['Day'], rldf['Hour'])
2621
+ return make_geos_date_ranges(rl_dates)
2622
+
2623
+
2624
+ def compute_file_checksum(file_path):
2625
+ """Compute an MD5 checksum on the given file
2626
+
2627
+ Parameters
2628
+ ----------
2629
+ file_path : str or Path
2630
+ Path to the file
2631
+
2632
+ Returns
2633
+ -------
2634
+ str
2635
+ The hexadecimal representation of the checksum
2636
+ """
2637
+ checksum = md5()
2638
+ with open(file_path, 'rb') as f:
2639
+ while True:
2640
+ block = f.read(1000000)
2641
+ if len(block) == 0:
2642
+ break
2643
+ checksum.update(block)
2644
+ return checksum.hexdigest()
2645
+
2646
+
2647
+ def compute_bytes_checksum(bytes):
2648
+ """Compute an MD5 checksum on the given bytes
2649
+
2650
+ Parameters
2651
+ ----------
2652
+ bytes
2653
+ A byte string. To compute the checksum of a string,
2654
+ use `.encode()` to convert it to bytes.
2655
+
2656
+ Returns
2657
+ -------
2658
+ str
2659
+ The hexadecimal representation of the checksum
2660
+ """
2661
+ checksum = md5()
2662
+ checksum.update(bytes)
2663
+ return checksum.hexdigest()