ginput 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ginput/__init__.py +4 -0
- ginput/common_utils/__init__.py +0 -0
- ginput/common_utils/ggg_logging.py +62 -0
- ginput/common_utils/ioutils.py +304 -0
- ginput/common_utils/mod_constants.py +24 -0
- ginput/common_utils/mod_utils.py +2663 -0
- ginput/common_utils/readers.py +477 -0
- ginput/common_utils/run_utils.py +57 -0
- ginput/common_utils/sat_utils.py +67 -0
- ginput/common_utils/versioning.py +251 -0
- ginput/common_utils/writers.py +377 -0
- ginput/data/.gitignore +1 -0
- ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
- ginput/data/ML_monthly_obs_ch4.txt +498 -0
- ginput/data/ML_monthly_obs_co.txt +412 -0
- ginput/data/ML_monthly_obs_co2.txt +597 -0
- ginput/data/ML_monthly_obs_n2o.txt +330 -0
- ginput/data/SMO_monthly_obs_ch4.txt +499 -0
- ginput/data/SMO_monthly_obs_co.txt +416 -0
- ginput/data/SMO_monthly_obs_co2.txt +613 -0
- ginput/data/SMO_monthly_obs_n2o.txt +330 -0
- ginput/data/ace_fn2o_lut.nc +0 -0
- ginput/data/age_spectra/age.midlat.txt +47 -0
- ginput/data/age_spectra/age.tropics.txt +46 -0
- ginput/data/age_spectra/age.vortex.txt +31 -0
- ginput/data/age_spectra/agespec.midlat.txt +47 -0
- ginput/data/age_spectra/agespec.tropics.txt +46 -0
- ginput/data/age_spectra/agespec.vortex.txt +31 -0
- ginput/data/age_spectra/time.txt +3001 -0
- ginput/data/ch4_hf_slopes.nc +0 -0
- ginput/data/ch4_strat_lut.nc +0 -0
- ginput/data/clams_age_clim_scaled.nc +0 -0
- ginput/data/co2_strat_lut.nc +0 -0
- ginput/data/fo2_data/.gitignore +2 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
- ginput/data/hf_strat_lut.nc +0 -0
- ginput/data/isotopologs.dat +199 -0
- ginput/data/meso_co_lut.nc +0 -0
- ginput/data/n2o_ch4_acefts.nc +0 -0
- ginput/data/n2o_strat_lut.nc +0 -0
- ginput/data/o2_mean_dmf.dat +52 -0
- ginput/data/summer_35N.vmr +182 -0
- ginput/data/washenfelder03_table_s3.txt +83 -0
- ginput/download/__init__.py +0 -0
- ginput/download/download_utils.py +46 -0
- ginput/download/get_GEOS5.py +367 -0
- ginput/download/get_MERRA2.py +177 -0
- ginput/download/get_NOAA_flask_data.py +103 -0
- ginput/download/get_fo2_data.py +140 -0
- ginput/mod_maker/__init__.py +0 -0
- ginput/mod_maker/mod_maker.py +2470 -0
- ginput/mod_maker/slantify.py +450 -0
- ginput/mod_maker/tccon_sites.py +570 -0
- ginput/priors/__init__.py +0 -0
- ginput/priors/acos_interface.py +1376 -0
- ginput/priors/automation.py +313 -0
- ginput/priors/backend_analysis/__init__.py +0 -0
- ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
- ginput/priors/backend_analysis/ace_fts_val.py +452 -0
- ginput/priors/backend_analysis/acos_comparison.py +550 -0
- ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
- ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
- ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
- ginput/priors/backend_analysis/backend_driver.py +90 -0
- ginput/priors/backend_analysis/backend_utils.py +358 -0
- ginput/priors/backend_analysis/clams.py +71 -0
- ginput/priors/backend_analysis/create_test_priors.py +853 -0
- ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
- ginput/priors/backend_analysis/mls_val.py +217 -0
- ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
- ginput/priors/fo2_prep.py +311 -0
- ginput/priors/map_maker.py +193 -0
- ginput/priors/mlo_smo_prep.py +1146 -0
- ginput/priors/tccon_priors.py +3482 -0
- ginput-1.3.0.dist-info/LICENSE.txt +203 -0
- ginput-1.3.0.dist-info/METADATA +125 -0
- ginput-1.3.0.dist-info/RECORD +86 -0
- ginput-1.3.0.dist-info/WHEEL +5 -0
- ginput-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2663 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Utility functions shared among the mod maker and prior generation code.
|
|
3
|
+
|
|
4
|
+
Compatibility notes:
|
|
5
|
+
* We have found that Pandas version 0.20 is incompatible with this module. Pandas >= 0.24 works.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import print_function, division
|
|
9
|
+
|
|
10
|
+
import datetime as dt
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from glob import glob
|
|
13
|
+
from hashlib import md5
|
|
14
|
+
|
|
15
|
+
import xarray as xr
|
|
16
|
+
from dateutil.relativedelta import relativedelta
|
|
17
|
+
import netCDF4 as ncdf
|
|
18
|
+
import numpy as np
|
|
19
|
+
from numpy import ma
|
|
20
|
+
import os
|
|
21
|
+
import pandas as pd
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
from numpy.core._multiarray_umath import arctan, tan, sin, cos
|
|
25
|
+
from scipy.interpolate import interp1d, interp2d
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
from warnings import warn
|
|
29
|
+
|
|
30
|
+
from . import mod_constants as const
|
|
31
|
+
from .mod_constants import days_per_year
|
|
32
|
+
from ..mod_maker.tccon_sites import site_dict
|
|
33
|
+
|
|
34
|
+
_std_model_pres_levels = np.array([1000.0, 975.0, 950.0, 925.0, 900.0, 875.0, 850.0, 825.0, 800.0, 775.0, 750.0, 725.0,
|
|
35
|
+
700.0, 650.0, 600.0, 550.0, 500.0, 450.0, 400.0, 350.0, 300.0, 250.0, 200.0, 150.0,
|
|
36
|
+
100.0, 70.0, 50.0, 40.0, 30.0, 20.0, 10.0, 7.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.7, 0.5,
|
|
37
|
+
0.4, 0.3, 0.1])
|
|
38
|
+
# as of 2019-03-14, both GEOS-FP and MERRA-2 use the same standard 42 pressure levels
|
|
39
|
+
# (c.f. https://gmao.gsfc.nasa.gov/GMAO_products/documents/GEOS_5_FP_File_Specification_ON4v1_2.pdf page 52 and
|
|
40
|
+
# https://gmao.gsfc.nasa.gov/reanalysis/MERRA/docs/MERRA_File_Specification.pdf page 27 for GEOS-FP and MERRA-2
|
|
41
|
+
# respectively)
|
|
42
|
+
merra_pres_levels = _std_model_pres_levels
|
|
43
|
+
geosfp_pres_levels = _std_model_pres_levels
|
|
44
|
+
earth_radius = 6371 # kilometers
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TropopauseError(Exception):
|
|
48
|
+
"""
|
|
49
|
+
Error if could not find the tropopause
|
|
50
|
+
"""
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ModelError(Exception):
|
|
55
|
+
"""
|
|
56
|
+
Error if a model file is nonsensical.
|
|
57
|
+
"""
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class GGGPathError(Exception):
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class ModPathError(IOError):
|
|
66
|
+
pass
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class ProgressBar(object):
|
|
70
|
+
"""
|
|
71
|
+
Create a text-based progress bar
|
|
72
|
+
|
|
73
|
+
An instance of this class can be used to print a text progress bar that does not need a new line for each progress
|
|
74
|
+
step. It uses carriage returns to reset to the beginning of each line before printing the next. This therefore
|
|
75
|
+
does not work well if other print statements occur in between calls to :meth:`print_bar`, the progress bar will
|
|
76
|
+
either end up on a new line anyway or potentially overwrite previous print statements if they did not end with a
|
|
77
|
+
newline.
|
|
78
|
+
|
|
79
|
+
:param num_symbols: how many steps there should be in the progress bar. In other words, the progress bar will be
|
|
80
|
+
complete when :meth:`print_bar` is called with ``num_symbols-1``.
|
|
81
|
+
:type num_symbols: int
|
|
82
|
+
|
|
83
|
+
:param prefix: a string to include before the beginning of each progress bar. The class will ensure that at least
|
|
84
|
+
one space is present between the prefix and the progress bar, but will not add one if one is already present at
|
|
85
|
+
the end of the prefix.
|
|
86
|
+
:type prefix: str
|
|
87
|
+
|
|
88
|
+
:param suffix: a string to include at the end of each progress bar. The class will ensure that at least one space
|
|
89
|
+
is present between the progress bar and the suffix, but will not add one if one is already present at the beginning
|
|
90
|
+
of the suffix.
|
|
91
|
+
:type suffix: str
|
|
92
|
+
|
|
93
|
+
:param add_one: if ``True``, the number of symbols printed in the progress bar is equal to ``i+1`` where ``i`` is
|
|
94
|
+
the argument to :meth:`print_bar`. This works well with Python loops over ``i in range(n)``, since the last value
|
|
95
|
+
of ``i`` will be ``n-1``, setting ``add_one`` to ``True`` ensures that a full progress bar is printed at the end.
|
|
96
|
+
:type add_one: bool
|
|
97
|
+
|
|
98
|
+
:param style: can be either '*' or 'counter'. The former prints a symbolic progress bar of the form:
|
|
99
|
+
|
|
100
|
+
[* ]
|
|
101
|
+
[** ]
|
|
102
|
+
[*** ]
|
|
103
|
+
[****]
|
|
104
|
+
|
|
105
|
+
where the number of *'s is set by ``num_symbols``. The latter will instead print 'i/num_symbols' for each step.
|
|
106
|
+
:type style: str
|
|
107
|
+
"""
|
|
108
|
+
def __init__(self, num_symbols, prefix='', suffix='', add_one=True, style='*'):
|
|
109
|
+
"""
|
|
110
|
+
See class help.
|
|
111
|
+
"""
|
|
112
|
+
if len(prefix) > 0 and not prefix.endswith(' '):
|
|
113
|
+
prefix += ' '
|
|
114
|
+
if len(suffix) > 0 and not suffix.startswith(' '):
|
|
115
|
+
suffix = ' ' + suffix
|
|
116
|
+
|
|
117
|
+
if style == '*':
|
|
118
|
+
self._fmt_str = '{pre}[{{pstr:<{n}}}]{suf}'.format(pre=prefix, n=num_symbols, suf=suffix)
|
|
119
|
+
elif style == 'counter':
|
|
120
|
+
self._fmt_str = '{pre}{{i:>{l}}}/{n}{suf}'.format(pre=prefix, n=num_symbols, suf=suffix, l=len(str(num_symbols)))
|
|
121
|
+
else:
|
|
122
|
+
raise ValueError('style "{}" not recognized'.format(style))
|
|
123
|
+
self._add_one = add_one
|
|
124
|
+
|
|
125
|
+
def print_bar(self, i):
|
|
126
|
+
"""
|
|
127
|
+
Print the iteration of the progress bar corresponding to step ``i``.
|
|
128
|
+
|
|
129
|
+
:param i: defines the progress step, either the number of *'s to print with ``style='*'`` or the counter number
|
|
130
|
+
with ``style='counter'``.
|
|
131
|
+
:type i: int
|
|
132
|
+
:return: None, prints to screen.
|
|
133
|
+
"""
|
|
134
|
+
if self._add_one:
|
|
135
|
+
i += 1
|
|
136
|
+
|
|
137
|
+
pstr = '*' * i
|
|
138
|
+
pbar = self._fmt_str.format(pstr=pstr, i=i)
|
|
139
|
+
sys.stdout.write('\r' + pbar)
|
|
140
|
+
sys.stdout.flush()
|
|
141
|
+
|
|
142
|
+
def finish(self):
|
|
143
|
+
"""
|
|
144
|
+
Close the progress bar. By default, just prints a newline.
|
|
145
|
+
:return: None
|
|
146
|
+
"""
|
|
147
|
+
sys.stdout.write('\n')
|
|
148
|
+
sys.stdout.flush()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def check_depedencies_newer(out_file, *dependency_files):
|
|
152
|
+
"""
|
|
153
|
+
Check if
|
|
154
|
+
:param out_file:
|
|
155
|
+
:param dependency_files:
|
|
156
|
+
:return:
|
|
157
|
+
"""
|
|
158
|
+
if len(dependency_files) == 0:
|
|
159
|
+
raise ValueError('Give at least one dependency file')
|
|
160
|
+
|
|
161
|
+
out_last_modified = os.path.getmtime(out_file)
|
|
162
|
+
for dep in dependency_files:
|
|
163
|
+
if os.path.getmtime(dep) > out_last_modified:
|
|
164
|
+
return True
|
|
165
|
+
|
|
166
|
+
return False
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def get_num_header_lines(filename):
|
|
170
|
+
"""
|
|
171
|
+
Get the number of header lines in a standard GGG file
|
|
172
|
+
|
|
173
|
+
This assumes that the file specified begins with a line with two numbers: the number of header rows and the number
|
|
174
|
+
of data columns.
|
|
175
|
+
|
|
176
|
+
:param filename: the file to read
|
|
177
|
+
:type filename: str
|
|
178
|
+
|
|
179
|
+
:return: the number of header lines
|
|
180
|
+
:rtype: int
|
|
181
|
+
"""
|
|
182
|
+
with open(filename, 'r') as fobj:
|
|
183
|
+
header_info = fobj.readline()
|
|
184
|
+
|
|
185
|
+
if ',' in header_info:
|
|
186
|
+
header = header_info.split(',')
|
|
187
|
+
else:
|
|
188
|
+
header = header_info.split()
|
|
189
|
+
return int(header[0])
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def get_isotopes_file(isotopes_file=None, use_gggpath=False):
|
|
193
|
+
"""
|
|
194
|
+
Get the path to the isotopologs.dat file
|
|
195
|
+
|
|
196
|
+
:param isotopes_file: user input path. If this is not None, it is returned after checking that it exists.
|
|
197
|
+
:type isotopes_file: str or None
|
|
198
|
+
|
|
199
|
+
:param use_gggpath: set to ``True`` to find the isotopologs.dat file at the location defined by the GGGPATH
|
|
200
|
+
environmental variable. If ``False`` and ``isotopes_file`` is ``None`` then the isotopologs.date file included in
|
|
201
|
+
this repo is used.
|
|
202
|
+
:type use_gggpath: bool
|
|
203
|
+
|
|
204
|
+
:return: the path to the isotopologs.dat file
|
|
205
|
+
:rtype: str
|
|
206
|
+
"""
|
|
207
|
+
if isotopes_file is not None:
|
|
208
|
+
if not os.path.isfile(isotopes_file):
|
|
209
|
+
raise IOError('The isotopes path {} is not a file'.format(isotopes_file))
|
|
210
|
+
return isotopes_file
|
|
211
|
+
elif use_gggpath:
|
|
212
|
+
gggpath = os.getenv('GGGPATH')
|
|
213
|
+
if gggpath is None:
|
|
214
|
+
raise EnvironmentError('use_gggpath=True requires the GGGPATH environmental variable to be set')
|
|
215
|
+
isotopes_file = os.path.join(gggpath, 'isotopologs', 'isotopologs.dat')
|
|
216
|
+
if not os.path.isfile(isotopes_file):
|
|
217
|
+
raise IOError('Failed to find isotopologs.dat at {}. Either update your GGGPATH environmental variable or '
|
|
218
|
+
'set use_gggpath to False.'.format(isotopes_file))
|
|
219
|
+
return isotopes_file
|
|
220
|
+
else:
|
|
221
|
+
return os.path.join(const.data_dir, 'isotopologs.dat')
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def mod_output_subdir(top_dir, site_abbrev, product='fpit', slant=False):
|
|
225
|
+
path_dir = 'slant' if slant else 'vertical'
|
|
226
|
+
return os.path.join(top_dir, product, site_abbrev, path_dir)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def vmr_output_subdir(top_dir, site_abbrev, product='fpit', slant=False):
|
|
230
|
+
path_dir = 'vmrs-slant' if slant else 'vmrs-vertical'
|
|
231
|
+
return os.path.join(top_dir, product, site_abbrev, path_dir)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def vmr_file_name(obs_date, lon, lat, keep_latlon_prec=False, date_fmt='%Y%m%d%H', in_utc=True):
|
|
235
|
+
"""
|
|
236
|
+
Construct the standard filename for a .vmr file produced by this code
|
|
237
|
+
|
|
238
|
+
:param obs_date: the datetime of the profiles
|
|
239
|
+
:type obs_date: datetime-like
|
|
240
|
+
|
|
241
|
+
:param lon: the longitude of the profiles.
|
|
242
|
+
:type lon: float
|
|
243
|
+
|
|
244
|
+
:param lat: the latitude of the profiles
|
|
245
|
+
:type lat: float
|
|
246
|
+
|
|
247
|
+
:param keep_latlon_prec: by default, lat and lon are rounded to the nearest whole number. Set this to ``True`` to
|
|
248
|
+
keep 2 decimal places of precision.
|
|
249
|
+
:type keep_latlon_prec: bool
|
|
250
|
+
|
|
251
|
+
:return: the .vmr file name, with format "JLv_yyyymmddhh_XX[NS]_YYY[EW].vmr" where "v" is the major version,
|
|
252
|
+
"yyyymmddhh" the date/time, XX[NS] the latitude and YYY[EW] the longitude.
|
|
253
|
+
:rtype: str
|
|
254
|
+
"""
|
|
255
|
+
# The desired behavior is to zero pad if rounding to whole degrees but not otherwise. This mimics the behavior for
|
|
256
|
+
# .mod files, although that function does it slightly differently, and reconciling the two should be done at some
|
|
257
|
+
# point.
|
|
258
|
+
prec = 2 if keep_latlon_prec else 0
|
|
259
|
+
pad = False if keep_latlon_prec else True
|
|
260
|
+
lat = format_lat(lat, prec=prec, zero_pad=pad)
|
|
261
|
+
lon = format_lon(lon, prec=prec, zero_pad=pad)
|
|
262
|
+
major_version = _get_priors_major_version()
|
|
263
|
+
return 'JL{ver}_{date}{tz}_{lat}_{lon}.vmr'.format(ver=major_version, date=obs_date.strftime(date_fmt),
|
|
264
|
+
tz='Z' if in_utc else 'L', lat=lat, lon=lon)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def vmr_file_name_from_mod(mod_file_name: str) -> str:
|
|
268
|
+
"""
|
|
269
|
+
Infer the base file name for a .vmr file given the corresponding .mod file.
|
|
270
|
+
|
|
271
|
+
:param mod_file_name: The base name (NOT the full path) of the .mod file. This must have the
|
|
272
|
+
where everything after the first underscore and leading up to the extension is shared with
|
|
273
|
+
the .vmr file name.
|
|
274
|
+
|
|
275
|
+
:returns: the base name for the .vmr file.
|
|
276
|
+
"""
|
|
277
|
+
# Assume the mod file name is something like "FPIT_2021042300Z_52N_000W.mod" and we want
|
|
278
|
+
# something like "JL1_2021042300Z_52N_000W.vmr". All we need do is replace the part before
|
|
279
|
+
# the first underscore.
|
|
280
|
+
date_loc = mod_file_name.split('_', maxsplit=1)[1].replace('.mod', '')
|
|
281
|
+
return 'JL{}_{}.vmr'.format(_get_priors_major_version(), date_loc)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _get_priors_major_version():
|
|
285
|
+
return const.priors_version.split('.')[0]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def map_file_name_from_mod_vmr_files(site_abbrev, mod_file, vmr_file, map_fmt):
|
|
289
|
+
if map_fmt not in {'nc', 'netcdf', 'txt', 'text'}:
|
|
290
|
+
raise ValueError(f'Unknown value for map_fmt: {map_fmt}')
|
|
291
|
+
is_ncdf = map_fmt in {'nc', 'netcdf'}
|
|
292
|
+
|
|
293
|
+
vmr_date = find_datetime_substring(os.path.basename(vmr_file), out_type=dt.datetime)
|
|
294
|
+
mod_date = find_datetime_substring(os.path.basename(mod_file), out_type=dt.datetime)
|
|
295
|
+
if vmr_date != mod_date:
|
|
296
|
+
raise RuntimeError('The .vmr and .mod files have different dates in their filenames!')
|
|
297
|
+
vmr_lonstr = find_lon_substring(os.path.basename(vmr_file))
|
|
298
|
+
mod_lonstr = find_lon_substring(os.path.basename(mod_file))
|
|
299
|
+
if vmr_lonstr != mod_lonstr:
|
|
300
|
+
raise RuntimeError('The .vmr and .mod files have different longitudes in their filenames!')
|
|
301
|
+
vmr_latstr = find_lat_substring(os.path.basename(vmr_file))
|
|
302
|
+
mod_latstr = find_lat_substring(os.path.basename(mod_file))
|
|
303
|
+
if vmr_latstr != mod_latstr:
|
|
304
|
+
raise RuntimeError('The .vmr and .mod files have different latitudes in their filenames!')
|
|
305
|
+
|
|
306
|
+
return '{site}_{lat}_{lon}_{date}Z.map{suffix}'.format(
|
|
307
|
+
site=site_abbrev, lat=vmr_latstr, lon=vmr_lonstr,
|
|
308
|
+
date=vmr_date.strftime('%Y%m%d%H'),
|
|
309
|
+
suffix='.nc' if is_ncdf else ''
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def format_lon(lon, prec=2, zero_pad=False):
|
|
314
|
+
"""
|
|
315
|
+
Convert longitude between string and numeric representations.
|
|
316
|
+
|
|
317
|
+
If ``lon`` is a number, then it is converted to a string. The string will be the absolute value with "W" or "E" at
|
|
318
|
+
the end to indicate west (<= 0) or east (> 0). If given a string in that format, it converts it to a number.
|
|
319
|
+
|
|
320
|
+
:param lon: the longitude to convert
|
|
321
|
+
:type lon: float or str
|
|
322
|
+
|
|
323
|
+
:param prec: the precision after the decimal point to use. Only has an effect when converting float to string.
|
|
324
|
+
:type prec: int
|
|
325
|
+
|
|
326
|
+
:param zero_pad: set to ``True`` to zero pad the longitude string so that there are 3 digits before the decimal
|
|
327
|
+
place. Only has an effect when converting float to string.
|
|
328
|
+
|
|
329
|
+
:return: the formatted longitude string or the float representation of the longitude, with west being negative.
|
|
330
|
+
:rtype: str or float
|
|
331
|
+
"""
|
|
332
|
+
def to_str(lon):
|
|
333
|
+
ew = 'E' if lon >= 0 else 'W'
|
|
334
|
+
# In Python float format specification, "0X.Y" means to zero pad so that there's X total characters and Y after
|
|
335
|
+
# the decimal point. We want lon zero padded to have three numbers before the decimal point, so the total width
|
|
336
|
+
# needs to be the precision + 4 if there will be a decimal point otherwise just 3.
|
|
337
|
+
width = prec + 4 if prec > 0 else prec + 3
|
|
338
|
+
pad = '0{}'.format(width) if zero_pad else ''
|
|
339
|
+
fmt_str = '{{:{padding}.{prec}f}}{{}}'.format(padding=pad, prec=prec)
|
|
340
|
+
return fmt_str.format(abs(lon), ew)
|
|
341
|
+
|
|
342
|
+
def to_float(lon):
|
|
343
|
+
if lon[-1] == 'E':
|
|
344
|
+
sign = 1
|
|
345
|
+
elif lon[-1] == 'W':
|
|
346
|
+
sign = -1
|
|
347
|
+
else:
|
|
348
|
+
raise ValueError('A longitude string must end in "E" or "W"')
|
|
349
|
+
|
|
350
|
+
return float(lon[:-1]) * sign
|
|
351
|
+
|
|
352
|
+
if isinstance(lon, str):
|
|
353
|
+
return to_float(lon)
|
|
354
|
+
else:
|
|
355
|
+
if lon > 180:
|
|
356
|
+
lon -= 360
|
|
357
|
+
return to_str(lon)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def find_lon_substring(string, to_float=False):
|
|
361
|
+
"""
|
|
362
|
+
Find a longitude substring in a string.
|
|
363
|
+
|
|
364
|
+
A longitude substring will match \d+[EW] or \d+\.\d+[EW].
|
|
365
|
+
|
|
366
|
+
:param string: the string to search for the longitude substring
|
|
367
|
+
:type string: str
|
|
368
|
+
|
|
369
|
+
:param to_float: when ``True``, converts the longitude to a float value using :func:`format_lon`, else returns the
|
|
370
|
+
string itself.
|
|
371
|
+
:type to_float: bool
|
|
372
|
+
|
|
373
|
+
:return: the longitude substring or float value
|
|
374
|
+
:rtype: str or float
|
|
375
|
+
"""
|
|
376
|
+
# search for one or more numbers, which may include a decimal point followed by at least one number then E or W.
|
|
377
|
+
lon_re = r'\d+(\.\d+)?[EW]'
|
|
378
|
+
lon_str = re.search(lon_re, string).group()
|
|
379
|
+
if to_float:
|
|
380
|
+
return format_lon(lon_str)
|
|
381
|
+
else:
|
|
382
|
+
return lon_str
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def format_lat(lat, prec=2, zero_pad=False):
|
|
386
|
+
"""
|
|
387
|
+
Convert latitude between string and numeric representations.
|
|
388
|
+
|
|
389
|
+
If ``lat`` is a number, then it is converted to a string. The string will be the absolute value with "N" or "S" at
|
|
390
|
+
the end to indicate south (<= 0) or north (> 0). If given a string in that format, it converts it to a number.
|
|
391
|
+
|
|
392
|
+
:param lat: the latitude to convert
|
|
393
|
+
:type lat: float or str
|
|
394
|
+
|
|
395
|
+
:param prec: the precision after the decimal point to use. Only has an effect when converting float to string.
|
|
396
|
+
:type prec: int
|
|
397
|
+
|
|
398
|
+
:param zero_pad: set to ``True`` to zero pad the latitude string so that there are 2 digits before the decimal
|
|
399
|
+
place. Only has an effect when converting float to string.
|
|
400
|
+
|
|
401
|
+
:return: the formatted latitude string or the float representation of the latitude, with south being negative.
|
|
402
|
+
:rtype: str or float
|
|
403
|
+
"""
|
|
404
|
+
def to_str(lat):
|
|
405
|
+
ns = 'N' if lat >= 0 else 'S'
|
|
406
|
+
# In Python float format specification, "0X.Y" means to zero pad so that there's X total characters and Y after
|
|
407
|
+
# the decimal point. We want lat zero padded to have two numbers before the decimal point, so the total width
|
|
408
|
+
# needs to be the precision + 3 if there will be a decimal point otherwise just 2.
|
|
409
|
+
width = prec + 3 if prec > 0 else prec + 2
|
|
410
|
+
pad = '0{}'.format(width) if zero_pad else ''
|
|
411
|
+
fmt_str = '{{:{padding}.{prec}f}}{{}}'.format(padding=pad, prec=prec)
|
|
412
|
+
return fmt_str.format(abs(lat), ns)
|
|
413
|
+
|
|
414
|
+
def to_float(lat):
|
|
415
|
+
if lat[-1] == 'N':
|
|
416
|
+
sign = 1
|
|
417
|
+
elif lat[-1] == 'S':
|
|
418
|
+
sign = -1
|
|
419
|
+
else:
|
|
420
|
+
raise ValueError('A latitude string must end in "N" or "S"')
|
|
421
|
+
|
|
422
|
+
return float(lat[:-1]) * sign
|
|
423
|
+
|
|
424
|
+
if isinstance(lat, str):
|
|
425
|
+
return to_float(lat)
|
|
426
|
+
else:
|
|
427
|
+
return to_str(lat)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def find_lat_substring(string, to_float=False):
|
|
431
|
+
"""
|
|
432
|
+
Find a latitude substring in a string.
|
|
433
|
+
|
|
434
|
+
A latitude substring will match \d+[NS] or \d+\.\d+[NS].
|
|
435
|
+
|
|
436
|
+
:param string: the string to search for the latitude substring
|
|
437
|
+
:type string: str
|
|
438
|
+
|
|
439
|
+
:param to_float: when ``True``, converts the latitude to a float value using :func:`format_lat`, else returns the
|
|
440
|
+
string itself.
|
|
441
|
+
:type to_float: bool
|
|
442
|
+
|
|
443
|
+
:return: the latitude substring or float value
|
|
444
|
+
:rtype: str or float
|
|
445
|
+
"""
|
|
446
|
+
# search for one or more numbers, which may include a decimal point followed by at least one number then N or S.
|
|
447
|
+
lat_re = r'\d+(\.\d+)?[NS]'
|
|
448
|
+
lat_str = re.search(lat_re, string).group()
|
|
449
|
+
if to_float:
|
|
450
|
+
return format_lat(lat_str)
|
|
451
|
+
else:
|
|
452
|
+
return lat_str
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def find_datetime_substring(string, out_type=str):
|
|
456
|
+
"""
|
|
457
|
+
Extract a date/time substring from a string.
|
|
458
|
+
|
|
459
|
+
This assumes that the date/time is formatted as %Y%m%d (YYYYMMDD) or %Y%m%d_%H%M (YYYYMMDD_hhmm).
|
|
460
|
+
|
|
461
|
+
:param string: the string to search for the date/time substring.
|
|
462
|
+
:type string: str
|
|
463
|
+
|
|
464
|
+
:param out_type: what type to return the date/time as. Default is to return the string. If another type is passed,
|
|
465
|
+
then it must have a ``strptime`` class method that accepts the string to parse and the format string as arguments,
|
|
466
|
+
i.e. it must behave like :func:`datetime.datetime.strptime`.
|
|
467
|
+
:type out_type: type
|
|
468
|
+
|
|
469
|
+
:return: the string or parsed datetime value
|
|
470
|
+
"""
|
|
471
|
+
|
|
472
|
+
# Search for YYYYMMDDHH or YYYYMMDD_HHMM preferentially but allow just YYYYMMDD
|
|
473
|
+
date_re = r'\d{8}(_\d{4}|\d{2})?'
|
|
474
|
+
date_str = re.search(date_re, string).group()
|
|
475
|
+
|
|
476
|
+
if out_type is str:
|
|
477
|
+
return date_str
|
|
478
|
+
else:
|
|
479
|
+
date_fmts = {8: '%Y%m%d', 10: '%Y%m%d%H', 13: '%Y%m%d_%H%M'}
|
|
480
|
+
date_fmt = date_fmts[len(date_str)]
|
|
481
|
+
return out_type.strptime(date_str, date_fmt)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _vcs_dir_helper(vcs_dir):
|
|
485
|
+
if vcs_dir is None:
|
|
486
|
+
vcs_dir = os.path.dirname(__file__)
|
|
487
|
+
if len(vcs_dir) == 0:
|
|
488
|
+
# If in the current directory, then dirname(__file__) gives an empty string, which isn't allowed as the argument
|
|
489
|
+
# to cwd in check_output
|
|
490
|
+
vcs_dir = '.'
|
|
491
|
+
return os.path.abspath(vcs_dir)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _is_git_repo(vcs_dir=None):
|
|
495
|
+
vcs_dir = _vcs_dir_helper(vcs_dir)
|
|
496
|
+
try:
|
|
497
|
+
subprocess.check_call(['git', 'status'], cwd=vcs_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
498
|
+
except subprocess.CalledProcessError:
|
|
499
|
+
return False
|
|
500
|
+
else:
|
|
501
|
+
return True
|
|
502
|
+
|
|
503
|
+
def vcs_commit_info(vcs_dir=None):
|
|
504
|
+
"""
|
|
505
|
+
Return the last commit hash on the current branch, current branch, and last commit date for a VCS repo.
|
|
506
|
+
|
|
507
|
+
This function will use Git if ``vcs_dir`` points to a valid Git repo and only fall back
|
|
508
|
+
on Mercurial if ``vcs_dir`` is not a Git repo.
|
|
509
|
+
|
|
510
|
+
:param vcs_dir: optional, the VCS directory to check. If not given, defaults to the one containing this repo.
|
|
511
|
+
:type vcs_dir: str
|
|
512
|
+
|
|
513
|
+
:return: the commit hash, branch name, and date. The date will be in the format e.g. "Mon Aug 01 18:20:15 2023 -0700",
|
|
514
|
+
i.e. "%a %b %d %H:%M:%S %Y %z" format in terms of strftime codes.
|
|
515
|
+
:rtype: str, str, str
|
|
516
|
+
"""
|
|
517
|
+
if _is_git_repo(vcs_dir):
|
|
518
|
+
return _git_commit_info(vcs_dir)
|
|
519
|
+
else:
|
|
520
|
+
return _hg_commit_info(vcs_dir)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _git_commit_info(git_dir=None):
|
|
524
|
+
git_dir = _vcs_dir_helper(git_dir)
|
|
525
|
+
# Get the last commit in the current branch
|
|
526
|
+
parent = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=git_dir).decode('utf8').strip()
|
|
527
|
+
# Get the current branch
|
|
528
|
+
branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=git_dir).decode('utf8').strip()
|
|
529
|
+
# And the date of the last commit. Git by default doesn't seem to zero pad the date,
|
|
530
|
+
# so force it to in order to match Mercurial
|
|
531
|
+
date = subprocess.check_output(['git', 'log', '-1', r'--format=%cd', r'--date=format:"%a %b %d %H:%M:%S %Y %z"'], cwd=git_dir).decode('utf8').strip()
|
|
532
|
+
|
|
533
|
+
return parent, branch, date
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _hg_commit_info(hg_dir=None):
|
|
537
|
+
hg_dir = _vcs_dir_helper(hg_dir)
|
|
538
|
+
|
|
539
|
+
# Get the last commit (-l 1) in the current branch (-f)
|
|
540
|
+
summary = subprocess.check_output(['hg', 'log', '-f', '-l', '1'], cwd=hg_dir).splitlines()
|
|
541
|
+
log_dict = dict()
|
|
542
|
+
# Since subprocess returns a bytes object (at least on Linux) rather than an encoded string object, all the strings
|
|
543
|
+
# below must be bytes, not unicode strings
|
|
544
|
+
for line in summary:
|
|
545
|
+
splitline = line.split(b':', 1)
|
|
546
|
+
if len(splitline) < 2:
|
|
547
|
+
continue
|
|
548
|
+
k, v = splitline
|
|
549
|
+
log_dict[k.strip()] = v.strip()
|
|
550
|
+
|
|
551
|
+
parent = re.search(b'(?<=:)\\w+', log_dict[b'changeset']).group()
|
|
552
|
+
# In Mercurial, if on the default branch, then log does not include a branch name in the output
|
|
553
|
+
branch = log_dict[b'branch'] if b'branch' in log_dict else b'default'
|
|
554
|
+
parent_date = log_dict[b'date']
|
|
555
|
+
# Convert to unicode strings to avoid them getting formatted as "b'abc'" or "b'default'" in unicode strings
|
|
556
|
+
return parent.decode('utf8'), branch.decode('utf8'), parent_date.decode('utf8')
|
|
557
|
+
|
|
558
|
+
def vcs_is_commit_clean(vcs_dir=None, ignore_untracked=True, ignore_files=tuple()):
|
|
559
|
+
"""
|
|
560
|
+
Checks if a VCS directory is clean.
|
|
561
|
+
|
|
562
|
+
By default, a directory is considered clean if all tracked files have no uncommitted changes. Untracked files are
|
|
563
|
+
not considered. Setting ``ignore_untracked`` to ``False`` means that there must be no untracked files for the
|
|
564
|
+
directory to be clean. This function will use Git if ``vcs_dir`` points to a valid Git repo and only fall back
|
|
565
|
+
on Mercurial if ``vcs_dir`` is not a Git repo.
|
|
566
|
+
|
|
567
|
+
:param vcs_dir: optional, the VCS directory to check. If not given, defaults to the one containing this repo.
|
|
568
|
+
:type vcs_dir: str
|
|
569
|
+
|
|
570
|
+
:param ignore_untracked: optional, set to ``False`` to require that there be no untracked files in the directory for
|
|
571
|
+
it to be considered clean.
|
|
572
|
+
:type ignore_untracked: bool
|
|
573
|
+
|
|
574
|
+
:param ignore_files: optional, a sequence of file paths (absolute or relative to the current working directory, not the
|
|
575
|
+
repo root) to ignore for the sake of determining if the repo is clean.
|
|
576
|
+
:type ignore_files: Sequence[str]
|
|
577
|
+
|
|
578
|
+
:return: ``True`` if the directory is clean, ``False`` otherwise.
|
|
579
|
+
:rtype: bool
|
|
580
|
+
"""
|
|
581
|
+
if _is_git_repo(vcs_dir):
|
|
582
|
+
return _git_is_commit_clean(vcs_dir, ignore_untracked=ignore_untracked, ignore_files=ignore_files)
|
|
583
|
+
else:
|
|
584
|
+
return _hg_is_commit_clean(vcs_dir, ignore_untracked=ignore_untracked, ignore_files=ignore_files)
|
|
585
|
+
|
|
586
|
+
def _in_vcs_ignore(f: str, root: str, ignore_files):
|
|
587
|
+
"""Returns True if ``f`` was in ``ignored_files``
|
|
588
|
+
|
|
589
|
+
``f`` must be a string that is a file path relative to ``root``, which is
|
|
590
|
+
the root of the VCS repo. ``ignore_files`` then is a sequence of strings
|
|
591
|
+
giving the list of files to ignore if they have changed since the last commit.
|
|
592
|
+
Note that ``ignore_files`` will *not* be relative to ``root``, they must be
|
|
593
|
+
absolute paths or relative to the current working directory.
|
|
594
|
+
"""
|
|
595
|
+
f = os.path.join(root, f)
|
|
596
|
+
for ignore in ignore_files:
|
|
597
|
+
if os.path.exists(ignore) and os.path.samefile(f, ignore):
|
|
598
|
+
return True
|
|
599
|
+
return False
|
|
600
|
+
|
|
601
|
+
def _git_is_commit_clean(git_dir=None, ignore_untracked=True, ignore_files=tuple()):
|
|
602
|
+
git_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).decode('utf8').strip()
|
|
603
|
+
summary = subprocess.check_output(['git', 'status', '--porcelain']).decode('utf8').strip().splitlines()
|
|
604
|
+
for line in summary:
|
|
605
|
+
stat, filename = line.split(maxsplit=2)
|
|
606
|
+
if stat == '??' and ignore_untracked:
|
|
607
|
+
continue
|
|
608
|
+
elif stat == '!!':
|
|
609
|
+
# ignored file according to the git status manpage?
|
|
610
|
+
continue
|
|
611
|
+
elif _in_vcs_ignore(filename, git_root, ignore_files):
|
|
612
|
+
continue
|
|
613
|
+
else:
|
|
614
|
+
return False
|
|
615
|
+
|
|
616
|
+
return True
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def _hg_is_commit_clean(hg_dir=None, ignore_untracked=True, ignore_files=tuple()):
|
|
620
|
+
"""
|
|
621
|
+
Checks if a mercurial directory is clean.
|
|
622
|
+
|
|
623
|
+
By default, a directory is considered clean if all tracked files have no uncommitted changes. Untracked files are
|
|
624
|
+
not considered. Setting ``ignore_untracked`` to ``False`` means that there must be no untracked files for the
|
|
625
|
+
directory to be clean.
|
|
626
|
+
|
|
627
|
+
:param hg_dir: optional, the mercurial directory to check. If not given, defaults to the one containing this repo.
|
|
628
|
+
:type hg_dir: str
|
|
629
|
+
|
|
630
|
+
:param ignore_untracked: optional, set to ``False`` to require that there be no untracked files in the directory for
|
|
631
|
+
it to be considered clean.
|
|
632
|
+
:type ignore_untracked: bool
|
|
633
|
+
|
|
634
|
+
:return: ``True`` if the directory is clean, ``False`` otherwise.
|
|
635
|
+
:rtype: bool
|
|
636
|
+
"""
|
|
637
|
+
hg_dir = _vcs_dir_helper(hg_dir)
|
|
638
|
+
hg_root = subprocess.check_output(['hg', 'root'], cwd=hg_dir).strip()
|
|
639
|
+
summary = subprocess.check_output(['hg', 'status'], cwd=hg_dir).splitlines()
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
# Since subprocess returns a bytes object (at least on Linux) rather than an encoded string object, all the strings
|
|
643
|
+
# below must be bytes, not unicode strings
|
|
644
|
+
for line in summary:
|
|
645
|
+
status, hg_file = [p.strip() for p in line.split(b' ', 1)]
|
|
646
|
+
if ignore_untracked and status == b'?':
|
|
647
|
+
pass
|
|
648
|
+
elif _in_vcs_ignore(hg_file, hg_root, ignore_files):
|
|
649
|
+
pass
|
|
650
|
+
else:
|
|
651
|
+
return False
|
|
652
|
+
|
|
653
|
+
return True
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _lrange(*args):
|
|
657
|
+
# Ensure Python 3 compatibility for adding range() calls together
|
|
658
|
+
r = range(*args)
|
|
659
|
+
if not isinstance(r, list):
|
|
660
|
+
r = list(r)
|
|
661
|
+
return r
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def round_to_zero(val):
|
|
665
|
+
sign = np.sign(val)
|
|
666
|
+
return np.floor(np.abs(val)) * sign
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
def iter_mod_dirs(mod_top_dir, path=None, check_if_exists=False):
|
|
670
|
+
"""
|
|
671
|
+
Iterate over available .mod site directories.
|
|
672
|
+
|
|
673
|
+
:param mod_top_dir: the root directory for the .mod files. Must include subdirectories that are the two-letter site
|
|
674
|
+
abbreviations.
|
|
675
|
+
:type mod_top_dir: str
|
|
676
|
+
|
|
677
|
+
:param path: the subpath within each of the site dirs to return. If None, then the site directory itself is
|
|
678
|
+
returned. Otherwise this is appended to the end and that returned.
|
|
679
|
+
:type path: str
|
|
680
|
+
|
|
681
|
+
:param check_if_exists: if ``True`` verify that the subpath exists before yielding. No effect if ``path`` is None.
|
|
682
|
+
:type check_if_exists: bool
|
|
683
|
+
|
|
684
|
+
:return: iterable of .mod file directories.
|
|
685
|
+
:rtype: iterable
|
|
686
|
+
"""
|
|
687
|
+
# only match two-character file names
|
|
688
|
+
site_dirs = sorted(glob(os.path.join(mod_top_dir, '??')))
|
|
689
|
+
for sdir in site_dirs:
|
|
690
|
+
if path is None:
|
|
691
|
+
yield sdir
|
|
692
|
+
else:
|
|
693
|
+
fullpath = os.path.join(sdir, path)
|
|
694
|
+
if check_if_exists and not os.path.isdir(fullpath):
|
|
695
|
+
raise ModPathError('Requested subpath "{}" does not exist in {}'.format(path, sdir))
|
|
696
|
+
yield fullpath
|
|
697
|
+
|
|
698
|
+
|
|
699
|
+
def extract_mod_site_abbrevs(mod_files, default='xx'):
|
|
700
|
+
"""
|
|
701
|
+
Extract the site abbreviations from .mod file paths
|
|
702
|
+
|
|
703
|
+
The .mod files no longer include the site abbreviation anywhere in the file name or header, so we need to extract
|
|
704
|
+
those from the directory paths. This function assumes paths of the format ".../abbr/<something>/*.mod", i.e. that
|
|
705
|
+
there are are least three levels (including the .mod file) and that the third from the end is the site abbreviation.
|
|
706
|
+
|
|
707
|
+
For each path, if (a) there are not three directory levels, (b) the third part from the end is not a two-letter
|
|
708
|
+
name, or (c) if it is a dictionary, this returns the default abbreviation.
|
|
709
|
+
|
|
710
|
+
:param mod_files: a single path to a .mod file, or a list of .mod files
|
|
711
|
+
:type mod_files: str or list(str)
|
|
712
|
+
|
|
713
|
+
:param default: the default site abbreviation to return if cannot find an abbreviation
|
|
714
|
+
:type default: str
|
|
715
|
+
|
|
716
|
+
:return: a single site abbreviation if ``mod_files`` is a string, a tuple of abbreviations otherwise.
|
|
717
|
+
:rtype: str or tuple(str)
|
|
718
|
+
"""
|
|
719
|
+
def extract_single(modf):
|
|
720
|
+
if isinstance(modf, dict):
|
|
721
|
+
return default
|
|
722
|
+
elif not isinstance(modf, str):
|
|
723
|
+
raise TypeError('Expected string or dictionary for each mod file')
|
|
724
|
+
|
|
725
|
+
fileparts = modf.split(os.sep)
|
|
726
|
+
if len(fileparts) < 3:
|
|
727
|
+
return default
|
|
728
|
+
elif not re.match(r'[a-zA-Z0-9]{2}', fileparts[-3]):
|
|
729
|
+
return default
|
|
730
|
+
else:
|
|
731
|
+
return fileparts[-3]
|
|
732
|
+
|
|
733
|
+
if isinstance(mod_files, str):
|
|
734
|
+
return extract_single(mod_files)
|
|
735
|
+
else:
|
|
736
|
+
return tuple([extract_single(modf) for modf in mod_files])
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def iter_mod_files(mod_top_dir, include_slant=False):
|
|
740
|
+
"""
|
|
741
|
+
Iterate over available .mod files.
|
|
742
|
+
|
|
743
|
+
:param mod_top_dir: the root directory for the .mod files. Must include subdirectories that are the two-letter site
|
|
744
|
+
abbreviations.
|
|
745
|
+
:type mod_top_dir: str
|
|
746
|
+
|
|
747
|
+
:param include_slant: not yet implemented, but in the future will turn on returning pairs of files (vertical and
|
|
748
|
+
slant).
|
|
749
|
+
:type include_slant: bool
|
|
750
|
+
|
|
751
|
+
:return: iterable of paths to .mod files. The paths will start with the given ``mod_top_dir``.
|
|
752
|
+
"""
|
|
753
|
+
if include_slant:
|
|
754
|
+
raise NotImplementedError('include_slant not yet implemented')
|
|
755
|
+
|
|
756
|
+
for vdir in iter_mod_dirs(mod_top_dir, 'vertical'):
|
|
757
|
+
vertical_files = sorted(glob(os.path.join(vdir, '*.mod')))
|
|
758
|
+
for f in vertical_files:
|
|
759
|
+
yield f
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def calculate_model_potential_temperature(temp, pres_levels=_std_model_pres_levels):
|
|
763
|
+
"""
|
|
764
|
+
Calculate potental temperature for model output on fixed pressure levels.
|
|
765
|
+
|
|
766
|
+
:param temp: The absolute temperature (in K) on the model grid.
|
|
767
|
+
:type temp: array-like
|
|
768
|
+
|
|
769
|
+
:param pres_levels: the pressure levels that the temperature is defined on. Must be a 1D vector, i.e. all columns in
|
|
770
|
+
the model must be on the same pressure levels. A standard set of pressure for GEOS FP is the default.
|
|
771
|
+
:type pres_levels: vector-like
|
|
772
|
+
|
|
773
|
+
:return: the potential temperature
|
|
774
|
+
"""
|
|
775
|
+
if temp.ndim != 4:
|
|
776
|
+
raise ValueError('temp expected to be 4D')
|
|
777
|
+
|
|
778
|
+
ntime, nlev, nlat, nlon = temp.shape
|
|
779
|
+
if nlev != pres_levels.size:
|
|
780
|
+
raise ValueError('Number of levels in temp != number of pressure levels defined')
|
|
781
|
+
pres = np.tile(pres_levels[np.newaxis, :, np.newaxis, np.newaxis], (ntime, 1, nlat, nlon))
|
|
782
|
+
return calculate_potential_temperature(pres, temp)
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def calculate_potential_temperature(pres, temp):
|
|
786
|
+
"""
|
|
787
|
+
Calculate potential temperature.
|
|
788
|
+
|
|
789
|
+
:param pres: Pressure in millibars/hPa.
|
|
790
|
+
:param temp: Absolute temperature in Kelvin.
|
|
791
|
+
:return: the potential temperature corresponding to those T/P coordinates.
|
|
792
|
+
"""
|
|
793
|
+
return temp * (1000/pres) ** 0.286
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def convert_geos_eta_coord(delp):
|
|
797
|
+
"""
|
|
798
|
+
Calculate the pressure grid for a GEOS native file.
|
|
799
|
+
|
|
800
|
+
:param pres: DELP (pressure thickness) array in Pa. May be any number of
|
|
801
|
+
dimensions, as long as exactly one has a length of 72.
|
|
802
|
+
:return: the pressure level midpoints, in hPa. Note the unit change, this
|
|
803
|
+
is because the GEOS DELP variable is usually in Pa, but hPa is the standard
|
|
804
|
+
unit for pressure levels in the Np files.
|
|
805
|
+
"""
|
|
806
|
+
dp_shape = np.array(delp.shape)
|
|
807
|
+
try:
|
|
808
|
+
i_ax = np.flatnonzero(dp_shape == 72).item()
|
|
809
|
+
except ValueError:
|
|
810
|
+
raise ValueError('delp is either missing its 72 level dimension or has multiple dimensions with length 72')
|
|
811
|
+
|
|
812
|
+
# From pg. 7 of the GEOS FP document (https://gmao.gsfc.nasa.gov/GMAO_products/documents/GEOS_5_FP_File_Specification_ON4v1_2.pdf)
|
|
813
|
+
# the top pressure is always 0.01 hPa. Since the columns are space-to-surface, we add the cumulative sum to get the
|
|
814
|
+
# level bottom pressure, then take the average along that axis to get the middle pressure
|
|
815
|
+
level_shape = dp_shape.copy()
|
|
816
|
+
level_shape[i_ax] = 1
|
|
817
|
+
top_p = 0.01
|
|
818
|
+
top_p_slice = np.full(level_shape, top_p)
|
|
819
|
+
|
|
820
|
+
delp = delp * 0.01 # assume input is in Pa, want hPa
|
|
821
|
+
p_edge = top_p + np.cumsum(delp, axis=i_ax)
|
|
822
|
+
p_edge = np.concatenate([top_p_slice, p_edge], axis=i_ax)
|
|
823
|
+
|
|
824
|
+
# Move the vertical axis to the front to do the averaging so we can just always average along the first dimension
|
|
825
|
+
p_edge = np.rollaxis(p_edge, i_ax, 0)
|
|
826
|
+
p_mid = 0.5 * (p_edge[:-1] + p_edge[1:])
|
|
827
|
+
return np.rollaxis(p_mid, i_ax, 0)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _construct_grid(*part_defs):
|
|
831
|
+
"""
|
|
832
|
+
Helper function to construct coordinates for a 1D grid
|
|
833
|
+
:param part_defs: a sequence of tuples (or lists) defining the start, stop, and step for each grid component. Each
|
|
834
|
+
tuple gets expanded as the arguments to :func:`numpy.arange`.
|
|
835
|
+
:return: the coordinates, sorted, and made sure to be unique
|
|
836
|
+
"""
|
|
837
|
+
grid_components = [np.arange(*part) for part in part_defs]
|
|
838
|
+
# Keep only unique values and sort them
|
|
839
|
+
return np.unique(np.concatenate(grid_components))
|
|
840
|
+
|
|
841
|
+
|
|
842
|
+
# Compute area of each grid cell and the total area
|
|
843
|
+
def calculate_area(lat, lon, lat_res=None, lon_res=None, muted=False):
|
|
844
|
+
"""
|
|
845
|
+
Calculate grid cell area for an equirectangular grid.
|
|
846
|
+
|
|
847
|
+
:param lat: the vector of grid cell center latitudes, in degrees
|
|
848
|
+
:param lon: the vector of grid cell center longitudes, in degrees
|
|
849
|
+
:param lat_res: the width of a single grid cell in the latitudinal direction, in degrees. If omitted, will be
|
|
850
|
+
calculated from the lat vector.
|
|
851
|
+
:param lon_res: the width of a single grid cell in the longitudinal direction, in degrees. If omitted, will be
|
|
852
|
+
calculated from the lat vector.
|
|
853
|
+
:return: 2D array of areas, in units of fraction of Earth's surface area.
|
|
854
|
+
"""
|
|
855
|
+
def calculate_resolution(coord_vec, coord_name):
|
|
856
|
+
res = np.diff(coord_vec)
|
|
857
|
+
if not np.all(res - res[0] < 0.001):
|
|
858
|
+
raise RuntimeError('Could not determine a unique {} resolution'.format(coord_name))
|
|
859
|
+
return res[0]
|
|
860
|
+
|
|
861
|
+
nlat = lat.size
|
|
862
|
+
nlon = lon.size
|
|
863
|
+
|
|
864
|
+
if lat_res is None:
|
|
865
|
+
lat_res = calculate_resolution(lat, 'lat')
|
|
866
|
+
if lon_res is None:
|
|
867
|
+
lon_res = calculate_resolution(lon, 'lon')
|
|
868
|
+
|
|
869
|
+
lat_half_res = 0.5 * lat_res
|
|
870
|
+
|
|
871
|
+
area = np.zeros([nlat, nlon])
|
|
872
|
+
|
|
873
|
+
for j in range(nlat):
|
|
874
|
+
Slat = lat[j]-lat_half_res
|
|
875
|
+
Nlat = lat[j]+lat_half_res
|
|
876
|
+
|
|
877
|
+
Slat = np.deg2rad(Slat)
|
|
878
|
+
Nlat = np.deg2rad(Nlat)
|
|
879
|
+
for i in range(nlon):
|
|
880
|
+
area[j, i] = np.deg2rad(lon_res)*np.abs(np.sin(Slat)-np.sin(Nlat))
|
|
881
|
+
|
|
882
|
+
if abs(np.sum(area) - 4*np.pi) > 0.0001: # ensure proper normalization so the total area of Earth is 4*pi
|
|
883
|
+
#if not muted:
|
|
884
|
+
# print('Total earth area is {:g} not 4pi (difference of {:g}), normalizing to 4pi.'
|
|
885
|
+
# .format(np.sum(area), np.sum(area) - 4*np.pi))
|
|
886
|
+
area *= 4*np.pi/np.sum(area)
|
|
887
|
+
|
|
888
|
+
return area
|
|
889
|
+
|
|
890
|
+
|
|
891
|
+
def calculate_eq_lat_on_grid(EPV, PT, area):
|
|
892
|
+
"""
|
|
893
|
+
Calculate equivalent latitude on a 4D grid.
|
|
894
|
+
|
|
895
|
+
:param EPV: the potential vorticity on the 4D grid.
|
|
896
|
+
:type EPV: :class:`numpy.ndarray`
|
|
897
|
+
|
|
898
|
+
:param PT: the potential temperature on the 4D grid.
|
|
899
|
+
:type PT: :class:`numpy.ndarray`
|
|
900
|
+
|
|
901
|
+
:param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
|
|
902
|
+
:type area: :class:`numpy.ndarray`
|
|
903
|
+
|
|
904
|
+
:return: equivalent latitude
|
|
905
|
+
:rtype: :class:`numpy.ndarray`
|
|
906
|
+
"""
|
|
907
|
+
EL = np.full_like(PT, np.nan)
|
|
908
|
+
|
|
909
|
+
for itime in range(PT.shape[0]):
|
|
910
|
+
interpolator = calculate_eq_lat(EPV[itime], PT[itime], area)
|
|
911
|
+
# This is probably going to be horrifically slow - but interp2d sometimes gives weird results when called with
|
|
912
|
+
# vectors, so unfortunately we have to call this one element at a time
|
|
913
|
+
pbar = ProgressBar(PT[itime].size, prefix='Calculating eq. lat for time {}/{}:'.format(itime, PT.shape[0]),
|
|
914
|
+
style='counter')
|
|
915
|
+
for i in range(PT[itime].size):
|
|
916
|
+
pbar.print_bar(i)
|
|
917
|
+
ilev, ilat, ilon = np.unravel_index(i, PT[itime].shape)
|
|
918
|
+
this_pt = PT[itime, ilev, ilat, ilon]
|
|
919
|
+
this_epv = EPV[itime, ilev, ilat, ilon]
|
|
920
|
+
EL[itime, ilev, ilat, ilon] = interpolator(this_epv, this_pt)[0]
|
|
921
|
+
|
|
922
|
+
return EL
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def calculate_eq_lat(EPV, PT, area):
|
|
926
|
+
"""
|
|
927
|
+
Construct an interpolator for equivalent latitude.
|
|
928
|
+
|
|
929
|
+
:param EPV: a 3D grid of potential vorticity
|
|
930
|
+
:type EPV: :class:`numpy.ndarray`
|
|
931
|
+
|
|
932
|
+
:param PT: a 3D grid of potential temperature
|
|
933
|
+
:type PT: :class:`numpy.ndarray`
|
|
934
|
+
|
|
935
|
+
:param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
|
|
936
|
+
:type area: :class:`numpy.ndarray`
|
|
937
|
+
|
|
938
|
+
:return: a 2D interpolator for equivalent latitude, requires potential vorticity and potential temperature as inputs
|
|
939
|
+
:rtype: :class:`scipy.interpolate.interp2d`
|
|
940
|
+
|
|
941
|
+
Note: when querying the interpolator for equivalent latitude, it is often best to call it with scalar values, even
|
|
942
|
+
though that is slower than calling it with the full vector of PV and PT that you wish to get EL for. The problem is
|
|
943
|
+
that scipy 2D interpolators, when given vectors as input, return a grid. This would be fine, except that the values
|
|
944
|
+
corresponding to the vector of PV and PT are not always along the diagonal and so cannot be extracted with
|
|
945
|
+
:func:`numpy.diag`. (I suspect what is happening is that the interpolator sorts the input values when constructing
|
|
946
|
+
the grid, but I have not tested this. -JLL)
|
|
947
|
+
"""
|
|
948
|
+
nlev, nlat, nlon = PT.shape
|
|
949
|
+
# Get rid of fill values, this fills the bottom of profiles with the first valid value
|
|
950
|
+
PT[PT > 1e4] = np.nan
|
|
951
|
+
EPV[EPV > 1e8] = np.nan
|
|
952
|
+
for i in range(nlat):
|
|
953
|
+
pd.DataFrame(PT[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
|
|
954
|
+
pd.DataFrame(EPV[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
|
|
955
|
+
|
|
956
|
+
# Define a fixed potential temperature grid, with increasing spacing
|
|
957
|
+
# this is done arbitrarily to get sufficient levels for the interpolation to work well, and not too much for the
|
|
958
|
+
# computations to take less time
|
|
959
|
+
if np.min(PT) > 300 or np.max(PT) < 1000:
|
|
960
|
+
raise ValueError('Potential temperature range is smaller than the [300, 1000] K assumed to create the '
|
|
961
|
+
'interpolation grid')
|
|
962
|
+
|
|
963
|
+
theta_grid = _construct_grid((round_to_zero(np.nanmin(PT)), 300.0, 2), (300.0, 350.0, 5.0), (350.0, 500.0, 10.0),
|
|
964
|
+
(500.0, 750.0, 20.0), (750.0, 1000.0, 30.0), (1000.0, round_to_zero(np.nanmax(PT)), 100.0))
|
|
965
|
+
new_nlev = np.size(theta_grid)
|
|
966
|
+
|
|
967
|
+
# Get PV on the fixed PT levels ~ 2 seconds per date
|
|
968
|
+
new_EPV = np.zeros([new_nlev, nlat, nlon])
|
|
969
|
+
for i in range(nlat):
|
|
970
|
+
for j in range(nlon):
|
|
971
|
+
new_EPV[:, i, j] = np.interp(theta_grid, PT[:, i, j], EPV[:, i, j])
|
|
972
|
+
|
|
973
|
+
# Compute equivalent latitudes
|
|
974
|
+
EL = np.zeros([new_nlev, 100])
|
|
975
|
+
EPV_thresh = np.zeros([new_nlev, 100])
|
|
976
|
+
for k in range(new_nlev): # loop over potential temperature levels
|
|
977
|
+
maxPV = np.max(new_EPV[k]) # global max PV
|
|
978
|
+
minPV = np.min(new_EPV[k]) # global min PV
|
|
979
|
+
|
|
980
|
+
# define 100 PV values between the min and max PV
|
|
981
|
+
EPV_thresh[k] = np.linspace(minPV,maxPV,100)
|
|
982
|
+
|
|
983
|
+
for l,thresh in enumerate(EPV_thresh[k]):
|
|
984
|
+
area_total = np.sum(area[new_EPV[k]>=thresh])
|
|
985
|
+
x = 1-area_total/(2*np.pi)
|
|
986
|
+
|
|
987
|
+
# With Python 3.10 and those dependencies, I started getting cases where x was *just* outside the -1 to 1 allowed domain,
|
|
988
|
+
# which led to NaNs in the EqL and so bad things downstream. Since values were only slightly outside the domain, it's fine
|
|
989
|
+
# to clip them, but if they go too far outside the expected values, then we may have a bigger problem.
|
|
990
|
+
if x < -1.01 or x > 1.01:
|
|
991
|
+
warn(f'Total area divided by 2*pi (x={x}) is far outside the domain of arcsin in EqL calculation. Clipping to -1 to 1.')
|
|
992
|
+
|
|
993
|
+
if x < -1:
|
|
994
|
+
x = -1
|
|
995
|
+
elif x > 1:
|
|
996
|
+
x = 1
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
EL[k,l] = np.arcsin(x)*90.0*2/np.pi
|
|
1000
|
+
|
|
1001
|
+
# Define a fixed potential vorticity grid, with increasing spacing away from 0
|
|
1002
|
+
# The last term should ensure that 0 is in the grid
|
|
1003
|
+
pv_grid = _construct_grid((round_to_zero(np.nanmin(EPV_thresh-50.0)), -1000.0, 50.0), (-1000.0, -500.0, 20.0), (-500.0, -100.0, 10.0),
|
|
1004
|
+
(-100.0, -10.0, 1.0), (-10.0, -1.0, 0.1), (-1.0, 1.0, 0.01), (1.0, 10.0, 0.1),
|
|
1005
|
+
(10.0, 100.0, 1.0), (100.0, 500.0, 10.0), (500.0, 1000.0, 20.0),
|
|
1006
|
+
(1000.0, round_to_zero(np.nanmax(EPV_thresh))+50.0, 50.0), (0.0, 0.1))
|
|
1007
|
+
|
|
1008
|
+
# Generate interpolating function to get EL for a given PV and PT
|
|
1009
|
+
interp_EL = np.zeros([new_nlev,len(pv_grid)])
|
|
1010
|
+
for k in range(new_nlev):
|
|
1011
|
+
interp_EL[k] = np.interp(pv_grid,EPV_thresh[k],EL[k])
|
|
1012
|
+
|
|
1013
|
+
return interp2d(pv_grid, theta_grid, interp_EL)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def get_eqlat_profile(interpolator, epv, theta):
|
|
1017
|
+
el = np.full_like(epv, np.nan)
|
|
1018
|
+
for i, (pv, pt) in enumerate(zip(epv, theta)):
|
|
1019
|
+
el[i] = interpolator(pv, pt)
|
|
1020
|
+
|
|
1021
|
+
return el
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def calculate_eq_lat_field(EPV, PT, area):
|
|
1025
|
+
"""
|
|
1026
|
+
Calculate 3D equivalent latitude from same shape potential vorticity and potiential temperature fields
|
|
1027
|
+
|
|
1028
|
+
:param EPV: a 3D grid of Ertel's potential vorticity
|
|
1029
|
+
:type EPV: :class:`numpy.ndarray`
|
|
1030
|
+
|
|
1031
|
+
:param PT: a 3D grid of potential temperature
|
|
1032
|
+
:type PT: :class:`numpy.ndarray`
|
|
1033
|
+
|
|
1034
|
+
:param area: the 2D grid of surface area (in steradians) that corresponds to the 2D slices of the 4D grid.
|
|
1035
|
+
:type area: :class:`numpy.ndarray`
|
|
1036
|
+
|
|
1037
|
+
:param method: the interpolation method used by griddata, one of ['linear','nearest','cubic']
|
|
1038
|
+
:type method: str
|
|
1039
|
+
|
|
1040
|
+
:return: the 3D equivalent latitude
|
|
1041
|
+
:rtype: :class:`numpy.ndarray`
|
|
1042
|
+
"""
|
|
1043
|
+
# scipy/dask import here to avoid making it a requirement in setup.py as this is not the main/standard function
|
|
1044
|
+
import dask
|
|
1045
|
+
import dask.array as da
|
|
1046
|
+
from dask.diagnostics import ProgressBar as dask_progress
|
|
1047
|
+
from scipy.interpolate import griddata
|
|
1048
|
+
|
|
1049
|
+
nlev, nlat, nlon = PT.shape
|
|
1050
|
+
# Get rid of fill values, this fills the bottom of profiles with the first valid value
|
|
1051
|
+
PT[PT > 1e4] = np.nan
|
|
1052
|
+
EPV[EPV > 1e8] = np.nan
|
|
1053
|
+
for i in range(nlat):
|
|
1054
|
+
pd.DataFrame(PT[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
|
|
1055
|
+
pd.DataFrame(EPV[:, i, :]).fillna(method='bfill', axis=0, inplace=True)
|
|
1056
|
+
|
|
1057
|
+
# Define a fixed potential temperature grid, with increasing spacing
|
|
1058
|
+
# this is done arbitrarily to get sufficient levels for the interpolation to work well, and not too much for the
|
|
1059
|
+
# computations to take less time
|
|
1060
|
+
if np.min(PT) > 300 or np.max(PT) < 1000:
|
|
1061
|
+
raise ValueError('Potential temperature range is smaller than the [300, 1000] K assumed to create the '
|
|
1062
|
+
'interpolation grid')
|
|
1063
|
+
|
|
1064
|
+
theta_array = _construct_grid((round_to_zero(np.nanmin(PT)), 300.0, 2), (300.0, 350.0, 5.0), (350.0, 500.0, 10.0),
|
|
1065
|
+
(500.0, 750.0, 20.0), (750.0, 1000.0, 30.0), (1000.0, round_to_zero(np.nanmax(PT)), 100.0))
|
|
1066
|
+
#theta_array = np.geomspace(round_to_zero(np.nanmin(PT)),round_to_zero(np.nanmax(PT)),130)
|
|
1067
|
+
new_nlev = np.size(theta_array)
|
|
1068
|
+
|
|
1069
|
+
# Get PV on the fixed PT levels ~ 2 seconds per date
|
|
1070
|
+
new_EPV = np.zeros([new_nlev, nlat, nlon])
|
|
1071
|
+
for i in range(nlat):
|
|
1072
|
+
for j in range(nlon):
|
|
1073
|
+
new_EPV[:, i, j] = np.interp(theta_array, PT[:, i, j], EPV[:, i, j])
|
|
1074
|
+
|
|
1075
|
+
# Compute equivalent latitudes
|
|
1076
|
+
EL = np.zeros([new_nlev, 100])
|
|
1077
|
+
EPV_thresh = np.zeros([new_nlev, 100])
|
|
1078
|
+
for k in range(new_nlev): # loop over potential temperature levels
|
|
1079
|
+
maxPV = np.max(new_EPV[k]) # global max PV
|
|
1080
|
+
minPV = np.min(new_EPV[k]) # global min PV
|
|
1081
|
+
|
|
1082
|
+
# define 100 PV values between the min and max PV
|
|
1083
|
+
#EPV_thresh[k] = np.unique(np.concatenate([-1.0*np.geomspace(1,np.abs(minPV),50),[0],np.geomspace(1,maxPV,50)]))
|
|
1084
|
+
EPV_thresh[k] = np.linspace(minPV,maxPV,100)
|
|
1085
|
+
|
|
1086
|
+
for l,thresh in enumerate(EPV_thresh[k]):
|
|
1087
|
+
area_total = np.sum(area[new_EPV[k]>=thresh])
|
|
1088
|
+
x = 1-area_total/(2*np.pi)
|
|
1089
|
+
|
|
1090
|
+
# see comment in calculate_eq_lat
|
|
1091
|
+
if x < -1.01 or x > 1.01:
|
|
1092
|
+
warn(f'Total area divided by 2*pi (x={x}) is far outside the domain of arcsin in EqL calculation. Clipping to -1 to 1.')
|
|
1093
|
+
|
|
1094
|
+
if x < -1:
|
|
1095
|
+
x = -1
|
|
1096
|
+
elif x > 1:
|
|
1097
|
+
x = 1
|
|
1098
|
+
|
|
1099
|
+
EL[k,l] = np.arcsin(x)*90.0*2/np.pi
|
|
1100
|
+
|
|
1101
|
+
# pv_array has much less elements than in calculate_eq_lat in order to be used with griddata
|
|
1102
|
+
pv_array = np.unique(np.concatenate([
|
|
1103
|
+
-1*np.geomspace(1e3,-1*round_to_zero(np.nanmin(EPV_thresh)),100),
|
|
1104
|
+
_construct_grid((-1000.0, -500.0, 20.0), (-500.0, -100.0, 10.0),(-100.0, -10.0, 1.0), (-10.0, -1.0, 0.1), (-1.0, 1.0, 0.01), (1.0, 10.0, 0.1),
|
|
1105
|
+
(10.0, 100.0, 1.0), (100.0, 500.0, 10.0), (500.0, 1000.0, 20.0), (0.0,0.1)),
|
|
1106
|
+
np.geomspace(1000,round_to_zero(np.nanmax(EPV_thresh)),100),
|
|
1107
|
+
]))
|
|
1108
|
+
|
|
1109
|
+
# Interpolate EL (equivalent latitude) on the fixed pv_array and theta_array
|
|
1110
|
+
interp_EL = np.zeros([new_nlev,len(pv_array)])
|
|
1111
|
+
for k in range(new_nlev):
|
|
1112
|
+
interp_EL[k] = np.interp(pv_array,EPV_thresh[k],EL[k])
|
|
1113
|
+
|
|
1114
|
+
pv_grid, theta_grid = np.meshgrid(pv_array,theta_array)
|
|
1115
|
+
|
|
1116
|
+
nonan = ~np.isnan(interp_EL) # there are no nans there, it's just to flatten the arrays
|
|
1117
|
+
flat_pv = pv_grid[nonan]
|
|
1118
|
+
flat_theta = theta_grid[nonan]
|
|
1119
|
+
flat_EL = interp_EL[nonan]
|
|
1120
|
+
|
|
1121
|
+
# Use griddata to compute the equivalent latitude level by level
|
|
1122
|
+
# compute the different levels in parallel with dask
|
|
1123
|
+
final_EL = []
|
|
1124
|
+
for ilev in range(nlev):
|
|
1125
|
+
final_EL.append(dask.delayed(griddata)((flat_pv, flat_theta), flat_EL, (EPV[ilev], PT[ilev])))
|
|
1126
|
+
|
|
1127
|
+
with dask_progress():
|
|
1128
|
+
final_EL = np.stack(dask.compute(*final_EL))
|
|
1129
|
+
|
|
1130
|
+
return final_EL
|
|
1131
|
+
|
|
1132
|
+
|
|
1133
|
+
def _format_geosfp_name(product, file_type, levels, date_time, add_subdir=False):
|
|
1134
|
+
"""
|
|
1135
|
+
Create the file name for a GEOS FP or FP-IT file.
|
|
1136
|
+
|
|
1137
|
+
:param product: which GEOS product ('fp' or 'fpit') to use
|
|
1138
|
+
|
|
1139
|
+
:param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
|
|
1140
|
+
:type file_type: str
|
|
1141
|
+
|
|
1142
|
+
:param levels: which levels ('surf', 'p', or 'eta') to use
|
|
1143
|
+
:type levels: str
|
|
1144
|
+
|
|
1145
|
+
:param date_time: the date and time of the desired file. The hour should be a multiple of 3.
|
|
1146
|
+
:type date_time: datetime-like
|
|
1147
|
+
|
|
1148
|
+
:param add_subdir: if ``True``, then the correct subdirectory will be prepended.
|
|
1149
|
+
:type add_subdir: bool
|
|
1150
|
+
|
|
1151
|
+
:return: the file name
|
|
1152
|
+
:rtype: str
|
|
1153
|
+
"""
|
|
1154
|
+
if product in {'it'}:
|
|
1155
|
+
return _format_geosit_name(product, file_type, levels, date_time, add_subdir=add_subdir)
|
|
1156
|
+
|
|
1157
|
+
product_patterns = {'fp': 'GEOS.fp.asm.inst3_{dim}d_{vars}_{type}.{date_time}.V01.nc4',
|
|
1158
|
+
'fpit': 'GEOS.fpit.asm.inst3_{dim}d_{vars}_{type}.GEOS5124.{date_time}.V01.nc4'}
|
|
1159
|
+
level_mapping = {'surf': 'Nx', 'p': 'Np', 'eta': 'Nv'}
|
|
1160
|
+
level_dims = {'Np': 3, 'Nx': 2, 'Nv': 3}
|
|
1161
|
+
var_types = {'met': 'asm', 'chm': 'chm'}
|
|
1162
|
+
try:
|
|
1163
|
+
pattern = product_patterns[product]
|
|
1164
|
+
except KeyError:
|
|
1165
|
+
raise ValueError('product "{}" has not been defined. Allowed values are: {}'
|
|
1166
|
+
.format(product, ', '.join(product_patterns.keys())))
|
|
1167
|
+
|
|
1168
|
+
try:
|
|
1169
|
+
levels = level_mapping[levels]
|
|
1170
|
+
except KeyError:
|
|
1171
|
+
raise ValueError('levels "{}" not recognized. Allowed values are: {}'
|
|
1172
|
+
.format(levels, ', '.join(level_mapping.keys())))
|
|
1173
|
+
|
|
1174
|
+
try:
|
|
1175
|
+
dims = level_dims[levels]
|
|
1176
|
+
except KeyError:
|
|
1177
|
+
raise ValueError('file_type "{}" is not recognized. Allowed values are: {}'
|
|
1178
|
+
.format(file_type, ', '.join(level_dims.keys())))
|
|
1179
|
+
|
|
1180
|
+
date_time = date_time.strftime('%Y%m%d_%H%M')
|
|
1181
|
+
fname = pattern.format(dim=dims, type=levels, date_time=date_time, vars=var_types[file_type])
|
|
1182
|
+
if add_subdir:
|
|
1183
|
+
fname = os.path.join(levels, fname)
|
|
1184
|
+
|
|
1185
|
+
return fname
|
|
1186
|
+
|
|
1187
|
+
def _format_geosit_name(product, file_type, levels, date_time, grid='L', add_subdir=False):
|
|
1188
|
+
product_patterns = {'it': 'GEOS.it.asm.{kind}_inst_{freq}hr_glo_{grid_key}_{lev}.GEOS5294.{dt:%Y-%m-%d}T{dt:%H%M}.V01.nc4'}
|
|
1189
|
+
|
|
1190
|
+
level_descr = {'surf': 'slv', 'eta': 'v72', 'p': 'p42'}
|
|
1191
|
+
level_freq = {'surf': 1, 'eta': 3, 'p': 3}
|
|
1192
|
+
level_subdirs = {'surf': 'Nx', 'eta': 'Nv', 'p': 'Np'}
|
|
1193
|
+
grid_keys = {'L': 'L576x361', 'C': 'C180x180x6'}
|
|
1194
|
+
var_types = {'met': 'asm', 'chm': 'chm'}
|
|
1195
|
+
|
|
1196
|
+
try:
|
|
1197
|
+
pattern = product_patterns[product]
|
|
1198
|
+
except KeyError:
|
|
1199
|
+
raise ValueError('product "{}" has not been defined. Allowed values are: {}'
|
|
1200
|
+
.format(product, ', '.join(product_patterns.keys())))
|
|
1201
|
+
|
|
1202
|
+
try:
|
|
1203
|
+
lev = level_descr[levels]
|
|
1204
|
+
except KeyError:
|
|
1205
|
+
raise ValueError('levels "{}" not recognized. Allowed values are: {}'
|
|
1206
|
+
.format(levels, ', '.join(level_descr.keys())))
|
|
1207
|
+
|
|
1208
|
+
try:
|
|
1209
|
+
freq = level_freq[levels]
|
|
1210
|
+
except KeyError:
|
|
1211
|
+
raise ValueError('level "{}" is not recognized. Allowed values are: {}'
|
|
1212
|
+
.format(levels, ', '.join(level_freq.keys())))
|
|
1213
|
+
|
|
1214
|
+
try:
|
|
1215
|
+
grid_key = grid_keys[grid]
|
|
1216
|
+
except KeyError:
|
|
1217
|
+
raise ValueError('grid "{}" is not recognized. Allowed values are: {}'
|
|
1218
|
+
.format(grid, ', '.join(grid_keys)))
|
|
1219
|
+
|
|
1220
|
+
try:
|
|
1221
|
+
var_type = var_types[file_type]
|
|
1222
|
+
except KeyError:
|
|
1223
|
+
raise ValueError('file_type "{}" is not recognized. Allowed values are: {}'
|
|
1224
|
+
.format(file_type, ', '.join(var_types)))
|
|
1225
|
+
|
|
1226
|
+
fname = pattern.format(kind=var_type, freq=freq, grid_key=grid_key, lev=lev, dt=date_time)
|
|
1227
|
+
if add_subdir:
|
|
1228
|
+
fname = os.path.join(level_subdirs[levels], fname)
|
|
1229
|
+
|
|
1230
|
+
return fname
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def read_geos_files(start_date, end_date, geos_path, profile_variables, surface_variables, product='fpit',
|
|
1234
|
+
keep_time_dim=True, concatenate_arrays=False, set_mask_to_nan=False):
|
|
1235
|
+
"""
|
|
1236
|
+
Read GEOS FP or FP-IT files between specified dates.
|
|
1237
|
+
|
|
1238
|
+
:param start_date: the first date to read GEOS files from
|
|
1239
|
+
:type start_date: datetime-like
|
|
1240
|
+
|
|
1241
|
+
:param end_date: the last date to read GEOS file from (exclusive). Note that if this datetime is not exactly a time
|
|
1242
|
+
that GEOS files are produced on, you will lose an extra file. For example, since GEOS files are produced every 3
|
|
1243
|
+
hours, if you specify ``end_date = datetime.datetime(2012, 1, 1, 23)``, you will lose the file from
|
|
1244
|
+
21:00 UTC 1 Jan 2012,
|
|
1245
|
+
:type end_date: datetime-like
|
|
1246
|
+
|
|
1247
|
+
:param geos_path: the path where the GEOS files are stored. Must have subdirectories 'Np' and 'Nx' for the profile
|
|
1248
|
+
and surface files, respectively. Currently, each of these subdirectories must be flat, meaning that all GEOS data
|
|
1249
|
+
from all times is stored at the top level, not organized into further subdirectories by year/month etc.
|
|
1250
|
+
:type geos_path: str
|
|
1251
|
+
|
|
1252
|
+
:param profile_variables: a list of variables to read from the profile (Np) files. 'lon', 'lat', and 'lev' are
|
|
1253
|
+
always read.
|
|
1254
|
+
:type profile_variables: list(str)
|
|
1255
|
+
|
|
1256
|
+
:param surface_variables: a list of variables to read from the surface (Nx) files. 'lon' and 'lat' are always read.
|
|
1257
|
+
:type surface_variables: list(str)
|
|
1258
|
+
|
|
1259
|
+
:param product: one of the strings 'fp' or 'fpit', determine which GEOS product is being read.
|
|
1260
|
+
:type product: str
|
|
1261
|
+
|
|
1262
|
+
:param keep_time_dim: Set to ``True`` to keep the time dimension of the variables. This means the profile and
|
|
1263
|
+
surface variables will be 4D and 3D respectively. Set to ``False`` to remove it (so they will be 3D and 2D).
|
|
1264
|
+
:type keep_time_dim: bool
|
|
1265
|
+
|
|
1266
|
+
:param concatenate_arrays: Set to ``True`` to concatenate the data from different files into a single array. This
|
|
1267
|
+
requires ``keep_time_dim`` to be ``True`` since they are concatenated along the time dimension. If ``False``, then
|
|
1268
|
+
the variables are left as lists of numpy arrays, where each array comes from a separate file.
|
|
1269
|
+
:type concatenate_arrays: bool
|
|
1270
|
+
|
|
1271
|
+
:param set_mask_to_nan: Set to ``True`` turn the masked arrays read from netCDF files by default into regular numpy
|
|
1272
|
+
arrays, where the masked values are replaced with NaNs.
|
|
1273
|
+
:type set_mask_to_nan: bool
|
|
1274
|
+
|
|
1275
|
+
:return: dictionaries of profile and surface variables, and a Pandas DateTimeIndex of the file dates. The variable
|
|
1276
|
+
dictionaries' values' format depends on the value of ``concatenate_arrays``.
|
|
1277
|
+
:rtype: dict, dict, DatetimeIndex
|
|
1278
|
+
"""
|
|
1279
|
+
def read_var_helper(nchandle, varname, keep_time=keep_time_dim):
|
|
1280
|
+
if keep_time:
|
|
1281
|
+
data = nchandle.variables[varname][:]
|
|
1282
|
+
else:
|
|
1283
|
+
# This is equivalent to doing nchandle.variables[varname][0,:,:,:] for a 4D variable; omitting the trailing
|
|
1284
|
+
# colons makes it general for any size array.
|
|
1285
|
+
data = nchandle.variables[varname][0]
|
|
1286
|
+
|
|
1287
|
+
if set_mask_to_nan:
|
|
1288
|
+
data = data.filled(np.nan)
|
|
1289
|
+
|
|
1290
|
+
return data
|
|
1291
|
+
|
|
1292
|
+
def read_files_helper(file_list, variables, is_profile):
|
|
1293
|
+
var_data = {v: [] for v in variables}
|
|
1294
|
+
for fidx, fname in enumerate(file_list):
|
|
1295
|
+
|
|
1296
|
+
with ncdf.Dataset(fname, 'r') as nchandle:
|
|
1297
|
+
# Always read lat/lon. If reading a profile file, get the levels too for the vertical coordinate.
|
|
1298
|
+
lon = read_var_helper(nchandle, 'lon', keep_time=True)
|
|
1299
|
+
lat = read_var_helper(nchandle, 'lat', keep_time=True)
|
|
1300
|
+
if is_profile:
|
|
1301
|
+
lev = read_var_helper(nchandle, 'lev', keep_time=True)
|
|
1302
|
+
|
|
1303
|
+
# If on the first file, store the coordinate variables. If on a later file, double check that the
|
|
1304
|
+
# coordinates are the same. They should be, and that assumption makes the data easier to work with
|
|
1305
|
+
# since we don't have to recheck our indices for each file.
|
|
1306
|
+
if fidx == 0:
|
|
1307
|
+
var_data['lon'] = lon
|
|
1308
|
+
var_data['lat'] = lat
|
|
1309
|
+
if is_profile:
|
|
1310
|
+
var_data['lev'] = lev
|
|
1311
|
+
else:
|
|
1312
|
+
chk = not ma.allclose(var_data['lon'], lon) or not ma.allclose(var_data['lat'], lat)
|
|
1313
|
+
if is_profile:
|
|
1314
|
+
chk = chk or not ma.allclose(var_data['lev'], lev)
|
|
1315
|
+
|
|
1316
|
+
if chk:
|
|
1317
|
+
# TODO: replace this with proper GEOS error from the backend analysis
|
|
1318
|
+
raise RuntimeError('lat, lon, and/or lev are inconsistent among the GEOS files')
|
|
1319
|
+
|
|
1320
|
+
for var in variables:
|
|
1321
|
+
var_data[var].append(read_var_helper(nchandle, var))
|
|
1322
|
+
|
|
1323
|
+
if concatenate_arrays:
|
|
1324
|
+
for var, data in var_data.items():
|
|
1325
|
+
if isinstance(data, list):
|
|
1326
|
+
# The lon/lat/lev variables don't need concatenated, we checked that they don't change with time
|
|
1327
|
+
# so there's only one array for them, not a list.
|
|
1328
|
+
var_data[var] = concatenate(data, axis=0)
|
|
1329
|
+
|
|
1330
|
+
return var_data
|
|
1331
|
+
|
|
1332
|
+
# input checking - we concatenate along the time dimension, so we better keep it
|
|
1333
|
+
if concatenate_arrays and not keep_time_dim:
|
|
1334
|
+
raise ValueError('concatenate_arrays = True requires keep_time_dim = True')
|
|
1335
|
+
|
|
1336
|
+
# If we're converting the default masked arrays to regular arrays, we have to use np.concatenate because
|
|
1337
|
+
# ma.concatenate always returns a masked array. If we're not converting, then the reverse applied.
|
|
1338
|
+
if set_mask_to_nan:
|
|
1339
|
+
concatenate = np.concatenate
|
|
1340
|
+
else:
|
|
1341
|
+
concatenate = ma.concatenate
|
|
1342
|
+
|
|
1343
|
+
geos_prof_files, file_dates = geosfp_file_names(product, 'met', 'p', start_date, end_date)
|
|
1344
|
+
geos_surf_files, surf_file_dates = geosfp_file_names(product, 'met', 'surf', start_date, end_date)
|
|
1345
|
+
|
|
1346
|
+
# Check that the file lists have the same dates
|
|
1347
|
+
if len(file_dates) != len(surf_file_dates) or any(file_dates[i] != surf_file_dates[i] for i in range(len(file_dates))):
|
|
1348
|
+
raise RuntimeError('Somehow listed different profile and surface files')
|
|
1349
|
+
elif concatenate_arrays:
|
|
1350
|
+
file_dates = pd.DatetimeIndex(file_dates)
|
|
1351
|
+
|
|
1352
|
+
geos_prof_files = [os.path.join(geos_path, 'Np', f) for f in geos_prof_files]
|
|
1353
|
+
geos_surf_files = [os.path.join(geos_path, 'Nx', f) for f in geos_surf_files]
|
|
1354
|
+
prof_data = read_files_helper(geos_prof_files, profile_variables, is_profile=True)
|
|
1355
|
+
surf_data = read_files_helper(geos_surf_files, surface_variables, is_profile=False)
|
|
1356
|
+
return prof_data, surf_data, file_dates
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
def geosfp_file_names(product, file_type, levels, start_date, end_date=None):
|
|
1360
|
+
"""
|
|
1361
|
+
List all file names for GEOS FP or FP-IT files for the given date(s).
|
|
1362
|
+
|
|
1363
|
+
:param product: which GEOS product ('fp' or 'fpit') to use
|
|
1364
|
+
:type product: str
|
|
1365
|
+
|
|
1366
|
+
:param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
|
|
1367
|
+
:type file_type: str
|
|
1368
|
+
|
|
1369
|
+
:param levels: which levels ('surf', 'p', or 'eta') to use
|
|
1370
|
+
:type levels: str
|
|
1371
|
+
|
|
1372
|
+
:param start_date: what date to start listing files for. If ``end_date`` is omitted, only the file for this date
|
|
1373
|
+
will be listed. Note that the hour must be a multiple of 3, since GEOS files are produced every three hours.
|
|
1374
|
+
:type start_date: datetime-like
|
|
1375
|
+
|
|
1376
|
+
:param end_date: what date to stop list files. This is exclusive, and will not itself be included. Can be omitted
|
|
1377
|
+
to just list one file.
|
|
1378
|
+
:type end_date: None or datetime-like
|
|
1379
|
+
|
|
1380
|
+
:return: the list of file names and an array of file dates
|
|
1381
|
+
:rtype: list, :class:`pandas.DatetimeIndex`
|
|
1382
|
+
"""
|
|
1383
|
+
freq = pd.Timedelta(hours=3)
|
|
1384
|
+
if start_date.hour % 3 != 0:
|
|
1385
|
+
raise ValueError('The hour of start_date must be a multiple of 3')
|
|
1386
|
+
if end_date is None:
|
|
1387
|
+
end_date = start_date + freq
|
|
1388
|
+
|
|
1389
|
+
geos_file_dates = pd.date_range(start=start_date, end=end_date - freq, freq=freq)
|
|
1390
|
+
geos_file_names = []
|
|
1391
|
+
for date in geos_file_dates:
|
|
1392
|
+
this_name = _format_geosfp_name(product, file_type, levels, date)
|
|
1393
|
+
geos_file_names.append(this_name)
|
|
1394
|
+
|
|
1395
|
+
return geos_file_names, geos_file_dates
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def geosfp_file_names_by_day(product, file_type, levels, utc_dates, utc_hours=None, add_subdir=False):
|
|
1399
|
+
"""
|
|
1400
|
+
Create a list of GEOS-FP file names for specified dates
|
|
1401
|
+
|
|
1402
|
+
This differs from :func:`geosfp_file_names` because this function can list files for only specific hours across
|
|
1403
|
+
multiple days. For example, if you want only 00:00 UTC FP-IT profile files for all of 2018, you would call this as::
|
|
1404
|
+
|
|
1405
|
+
geosfp_file_names_by_day('fpit', 'Np', pd.date_range('2018-01-01', '2018-12-31', utc_hours=[0])
|
|
1406
|
+
|
|
1407
|
+
:param product: which GEOS-FP product to make names for: "fp" or "fpit"
|
|
1408
|
+
:type product: str
|
|
1409
|
+
|
|
1410
|
+
:param file_type: which file type ('met' for meteorology, 'chm' for chemistry) to use
|
|
1411
|
+
:type file_type: str
|
|
1412
|
+
|
|
1413
|
+
:param levels: which levels ('surf', 'p', or 'eta') to use
|
|
1414
|
+
:type levels: str
|
|
1415
|
+
|
|
1416
|
+
:param utc_dates: Dates (on UTC time) to read files for.
|
|
1417
|
+
:type utc_dates: collection(datetime) or collection(datetime-like objects)
|
|
1418
|
+
|
|
1419
|
+
:param utc_hours: Which hours of the day to use (in UTC). If ``None``, then all hours that GEOS is produced on is
|
|
1420
|
+
used (every 3 hours). Otherwise, pass a collection of integers to specify a subset of hours to use. (e.g.
|
|
1421
|
+
[0, 3, 6, 9] to only use the files from the first half of each day).
|
|
1422
|
+
:type utc_hours: None or collection(int)
|
|
1423
|
+
|
|
1424
|
+
:param add_subdir: if ``True``, then the correct subdirectory will be prepended.
|
|
1425
|
+
:type add_subdir: bool
|
|
1426
|
+
|
|
1427
|
+
:return: a list of GEOS file names
|
|
1428
|
+
:rtype: list(str)
|
|
1429
|
+
"""
|
|
1430
|
+
geos_utc_hours = np.arange(0, 24, 3)
|
|
1431
|
+
if utc_hours is not None:
|
|
1432
|
+
geos_utc_hours = geos_utc_hours[np.isin(geos_utc_hours, utc_hours)]
|
|
1433
|
+
|
|
1434
|
+
geos_file_names = []
|
|
1435
|
+
geos_file_dates = []
|
|
1436
|
+
for date in utc_dates:
|
|
1437
|
+
for hr in geos_utc_hours:
|
|
1438
|
+
date_time = dt.datetime(date.year, date.month, date.day, hr)
|
|
1439
|
+
this_name = _format_geosfp_name(product, file_type, levels, date_time, add_subdir=add_subdir)
|
|
1440
|
+
geos_file_names.append(this_name)
|
|
1441
|
+
geos_file_dates.append(date_time)
|
|
1442
|
+
|
|
1443
|
+
return geos_file_names, geos_file_dates
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def datetime_from_geos_filename(geos_filename):
|
|
1447
|
+
geos_filename = os.path.basename(geos_filename)
|
|
1448
|
+
if geos_filename.startswith('GEOS.it'):
|
|
1449
|
+
date_str = re.search(r'\d{4}-\d{2}-\d{2}T\d{4}', geos_filename).group()
|
|
1450
|
+
return dt.datetime.strptime(date_str, '%Y-%m-%dT%H%M')
|
|
1451
|
+
else:
|
|
1452
|
+
date_str = re.search(r'\d{8}_\d{4}', geos_filename).group()
|
|
1453
|
+
return dt.datetime.strptime(date_str, '%Y%m%d_%H%M')
|
|
1454
|
+
|
|
1455
|
+
|
|
1456
|
+
def is_geos_on_native_grid(geos_filename):
|
|
1457
|
+
with ncdf.Dataset(geos_filename, 'r') as nch:
|
|
1458
|
+
return nch['lev'].size == 72
|
|
1459
|
+
|
|
1460
|
+
|
|
1461
|
+
def mode_to_product(mode):
|
|
1462
|
+
return mode.replace('-eta', '')
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
def mod_interpolation_legacy(z_grid, z_met, t_met, val_met, interp_mode=1, met_alt_geopotential=True):
|
|
1466
|
+
"""
|
|
1467
|
+
Legacy interpolation for .mod file profiles onto the TCCON grid
|
|
1468
|
+
|
|
1469
|
+
:param z_grid: the altitude levels (in kilometers) to interpolate the values onto
|
|
1470
|
+
:type z_grid: :class:`numpy.ndarray`
|
|
1471
|
+
|
|
1472
|
+
:param z_met: the altitude levels (in kilometers) of the input values
|
|
1473
|
+
:type z_met: :class:`numpy.ndarray`
|
|
1474
|
+
|
|
1475
|
+
:param t_met: the absolute temperature (in Kelvin) on the same levels as the input values
|
|
1476
|
+
:type t_met: :class:`numpy.ndarray`
|
|
1477
|
+
|
|
1478
|
+
:param val_met: the input values to be interpolated to the ``z_grid`` levels
|
|
1479
|
+
:type val_met: :class:`numpy.ndarray`
|
|
1480
|
+
|
|
1481
|
+
:param interp_mode: how to do the interpolation. Mode ``1`` (default) is used in the original GGG code for water
|
|
1482
|
+
vapor dry-air mole fraction. Recommended mode for anything relating to a concentration. Mode ``0`` is for
|
|
1483
|
+
temperature only. Mode ``2`` is for pressure, or more generally, values with an exponential dependence on altitude.
|
|
1484
|
+
:type interp_mode: int
|
|
1485
|
+
|
|
1486
|
+
:param met_alt_geopotential: if ``True``, in met altitudes are assumed to be heights based on geopotential, not
|
|
1487
|
+
geometry. Therefore internally, the grid altitudes are slightly modified to be compatible. If ``False``, then the
|
|
1488
|
+
met altitudes are assumed to be geometric, and the grid altitudes will not be scaled.
|
|
1489
|
+
:type met_alt_geopotential: bool
|
|
1490
|
+
|
|
1491
|
+
:return: the input values, interpolated onto the ``z_grid`` altitudes.
|
|
1492
|
+
:rtype: :class:`numpy.ndarray`
|
|
1493
|
+
"""
|
|
1494
|
+
if met_alt_geopotential:
|
|
1495
|
+
z_grid = z_grid / (1 + z_grid/earth_radius)
|
|
1496
|
+
|
|
1497
|
+
val_grid = np.full(z_grid.shape, np.nan, dtype=val_met.dtype)
|
|
1498
|
+
for i, z in enumerate(z_grid):
|
|
1499
|
+
# Find the levels in the met data above and below the current grid level. If we're below the first met level,
|
|
1500
|
+
# use the first two levels to extrapolate
|
|
1501
|
+
i_below = np.argwhere(z_met < z)
|
|
1502
|
+
if i_below.size == 0:
|
|
1503
|
+
i_below = 0
|
|
1504
|
+
else:
|
|
1505
|
+
i_below = np.max(i_below)
|
|
1506
|
+
|
|
1507
|
+
i_above = i_below + 1
|
|
1508
|
+
|
|
1509
|
+
# Calculate beta, which is used as the interpolation factor
|
|
1510
|
+
lapse_rate = (t_met[i_above] - t_met[i_below]) / (z_met[i_above] - z_met[i_below])
|
|
1511
|
+
if lapse_rate * (t_met[i_above] - t_met[i_below]) > 0.01 * t_met[i_below]:
|
|
1512
|
+
beta = np.log(1 + lapse_rate * (z - z_grid[i_below])/t_met[i_below]) / np.log(1 + lapse_rate*(z_met[i_below] - z_met[i_below]) / t_met[i_below])
|
|
1513
|
+
else:
|
|
1514
|
+
beta = (z - z_met[i_below]) / (z_met[i_above] - z_met[i_below])
|
|
1515
|
+
|
|
1516
|
+
# Different interpolation modes that come from the GGG2014 fortran code.
|
|
1517
|
+
if interp_mode == 0:
|
|
1518
|
+
# interp_mode = 0 is for temperature only, uses lapse rate directly
|
|
1519
|
+
val_grid[i] = val_met[i_below] + lapse_rate * (z - z_met[i_below])
|
|
1520
|
+
elif interp_mode == 1:
|
|
1521
|
+
# interp_mode = 1 is for species concentrations
|
|
1522
|
+
val_grid[i] = val_met[i_below] + beta * (val_met[i_above] - val_met[i_below])
|
|
1523
|
+
elif interp_mode == 2:
|
|
1524
|
+
# interp_mode = 2 is for pressure
|
|
1525
|
+
val_grid[i] = val_met[i_below] * (val_met[i_above] / val_met[i_below])**beta
|
|
1526
|
+
else:
|
|
1527
|
+
raise ValueError('interp_mode = {} is not allowed. Must be 0, 1, or 2'.format(interp_mode))
|
|
1528
|
+
|
|
1529
|
+
return val_grid
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def mod_interpolation_new(z_grid, z_met, vals_met, interp_mode='linear'):
|
|
1533
|
+
"""
|
|
1534
|
+
New method to interpolate met data onto the TCCON prior altitude grid
|
|
1535
|
+
|
|
1536
|
+
:param z_grid: the altitude levels to interpolate to.
|
|
1537
|
+
:type z_grid: :class:`numpy.ndarray`
|
|
1538
|
+
|
|
1539
|
+
:param z_met: the altitude levels in the meteorology data
|
|
1540
|
+
:type z_met: :class:`numpy.ndarray`
|
|
1541
|
+
|
|
1542
|
+
:param vals_met: the values to be interpolated, on the ``z_met`` levels
|
|
1543
|
+
:type vals_met: :class:`numpy.ndarray`
|
|
1544
|
+
|
|
1545
|
+
:param interp_mode: how to do the interpolation:
|
|
1546
|
+
|
|
1547
|
+
* ``'linear'`` will do linear interpolation of ``vals_met`` with respect to z.
|
|
1548
|
+
* ``'lin-log'`` linearly interpolate ln(``vals_met``) with respect to z.
|
|
1549
|
+
* ``'log-lin'`` linearly interpolated ``vals_met`` with respect to ln(z).
|
|
1550
|
+
* ``'log-log'`` linearly interpolate ln(``vals_met``) with respect to ln(z).
|
|
1551
|
+
|
|
1552
|
+
For compatibility with `mod_interpolation_legacy`, ``interp_mode`` may also be an integer. ``0`` or ``1`` are
|
|
1553
|
+
aliases for ``'linear'`` and ``2`` is the same as ``'lin-log'``
|
|
1554
|
+
:type interp_mode: str or int
|
|
1555
|
+
|
|
1556
|
+
:return: the values interpolated to the TCCON grid.
|
|
1557
|
+
:rtype: :class:`numpy.ndarray`
|
|
1558
|
+
"""
|
|
1559
|
+
interp_mode_compat_mapping = {0: 'linear', 1: 'linear', 2: 'lin-log', 3: 'log-lin', 4: 'log-log'}
|
|
1560
|
+
err_msg = 'interp_mode = {} is invalid. It must be one of the strings "{}" or one of the integers {}'.format(
|
|
1561
|
+
interp_mode, '", "'.join(interp_mode_compat_mapping.values()),
|
|
1562
|
+
', '.join([str(k) for k in interp_mode_compat_mapping.keys()])
|
|
1563
|
+
)
|
|
1564
|
+
if isinstance(interp_mode, int):
|
|
1565
|
+
try:
|
|
1566
|
+
interp_mode = interp_mode_compat_mapping[interp_mode]
|
|
1567
|
+
except KeyError:
|
|
1568
|
+
raise ValueError(err_msg)
|
|
1569
|
+
elif interp_mode not in interp_mode_compat_mapping.values():
|
|
1570
|
+
raise ValueError(err_msg)
|
|
1571
|
+
|
|
1572
|
+
interp_mode = interp_mode.lower()
|
|
1573
|
+
do_log_x = re.match('^log-\w{3}', interp_mode)
|
|
1574
|
+
do_log_y = re.match('\w{3}-log$', interp_mode)
|
|
1575
|
+
|
|
1576
|
+
if do_log_x:
|
|
1577
|
+
z_grid = np.log(z_grid)
|
|
1578
|
+
z_met = np.log(z_met)
|
|
1579
|
+
if do_log_y:
|
|
1580
|
+
vals_met = np.log(vals_met)
|
|
1581
|
+
|
|
1582
|
+
vals_interp = interp1d(z_met, vals_met, fill_value='extrapolate')
|
|
1583
|
+
vals_grid = vals_interp(z_grid)
|
|
1584
|
+
|
|
1585
|
+
# If doing logarithmic interpolation for the y variable need to restore the output values. (x not returned so no
|
|
1586
|
+
# need to restore.)
|
|
1587
|
+
if do_log_y:
|
|
1588
|
+
vals_grid = np.exp(vals_grid)
|
|
1589
|
+
|
|
1590
|
+
return vals_grid
|
|
1591
|
+
|
|
1592
|
+
|
|
1593
|
+
def interp_tropopause_height_from_pressure(p_trop_met, p_met, z_met):
|
|
1594
|
+
"""
|
|
1595
|
+
Calculate the tropopause height by interpolating to the tropopause pressure
|
|
1596
|
+
|
|
1597
|
+
:param p_trop_met: the blended tropopause pressure from GEOS Nx files.
|
|
1598
|
+
:type p_trop_met: float
|
|
1599
|
+
|
|
1600
|
+
:param p_met: the vector of pressure for this profile. Must be in the same units as ``p_trop_met``.
|
|
1601
|
+
:type p_met: array-like
|
|
1602
|
+
|
|
1603
|
+
:param z_met: the vector of altitude levels for this profile.
|
|
1604
|
+
:type z_met: array-like
|
|
1605
|
+
|
|
1606
|
+
:return: the tropopause altitude, in the same units as ``z_met``.
|
|
1607
|
+
:rtype: float
|
|
1608
|
+
"""
|
|
1609
|
+
# The age-of-air calculation used for the tropospheric trace gas profile calculation needs the tropopause altitude.
|
|
1610
|
+
# Previously we'd tried finding this by interpolating to the tropopause potential temperature, in order to be
|
|
1611
|
+
# consistent about defining the strat/trop separation by potential temperature. However, potential temperature
|
|
1612
|
+
# does not always behave in a manner that makes interpolating to it straightforward (e.g. it crosses the tropopause
|
|
1613
|
+
# theta 0 or >1 times) so we just use pressure now.
|
|
1614
|
+
z_trop_met = mod_interpolation_new(p_trop_met, p_met, z_met, 'log-lin')
|
|
1615
|
+
if z_trop_met < np.nanmin(z_met):
|
|
1616
|
+
raise RuntimeError('Tropopause altitude calculated to be below the bottom of the profile. Something has '
|
|
1617
|
+
'gone horribly wrong.')
|
|
1618
|
+
return z_trop_met
|
|
1619
|
+
|
|
1620
|
+
|
|
1621
|
+
def calc_wmo_tropopause(temperature, altitude, limit_to=(5., 18.), raise_error=True):
|
|
1622
|
+
"""
|
|
1623
|
+
Find the tropopause altitude using the WMO definition
|
|
1624
|
+
|
|
1625
|
+
The WMO thermal definition of the tropopause is: "the level at which the lapse rate drops to < 2 K/km and the
|
|
1626
|
+
average lapse rate between this and all higher levels within 2 km does not exceed 2 K/km".
|
|
1627
|
+
(quoted in https://www.atmos-chem-phys.net/8/1483/2008/acp-8-1483-2008.pdf, sect. 2.4).
|
|
1628
|
+
|
|
1629
|
+
:param temperature: the temperature profile, in K
|
|
1630
|
+
:type temperature: :class:`numpy.ndarray` (1D)
|
|
1631
|
+
|
|
1632
|
+
:param altitude: the altitude for each level in the temperature profile, in kilometers
|
|
1633
|
+
:type altitude: :class:`numpy.ndarray` (1D)
|
|
1634
|
+
|
|
1635
|
+
:param limit_to: the range of altitudes to limit the search for the tropopause to. This both helps avoid erroneous
|
|
1636
|
+
results and potentially speed up the analysis.
|
|
1637
|
+
:type limit_to: tuple(float, float)
|
|
1638
|
+
|
|
1639
|
+
:param raise_error: If ``True``, this function raises an error if it cannot find the tropopause. If ``False``, it
|
|
1640
|
+
returns a NaN in that case.
|
|
1641
|
+
:type raise_error: bool
|
|
1642
|
+
|
|
1643
|
+
:return: the tropopause altitude in kilometers
|
|
1644
|
+
:rtype: float
|
|
1645
|
+
:raises TropopauseError: if ``raise_error`` is ``True`` and this cannot find the tropopause.
|
|
1646
|
+
"""
|
|
1647
|
+
|
|
1648
|
+
# Calculate the lapse rate on the half levels. By definition, a positive lapse rate is a decrease with altitude, so
|
|
1649
|
+
# we need the minus sign.
|
|
1650
|
+
lapse = -np.diff(temperature) / np.diff(altitude)
|
|
1651
|
+
alt_half = altitude[:-1] + np.diff(altitude)/2.0
|
|
1652
|
+
|
|
1653
|
+
# Cut down the data to just the relevant range of altitudes recommended by
|
|
1654
|
+
# https://agupubs.onlinelibrary.wiley.com/doi/pdf/10.1029/2003GL018240 (end of sect. 2)
|
|
1655
|
+
zz = (alt_half >= np.min(limit_to)) & (alt_half <= np.max(limit_to))
|
|
1656
|
+
lapse = lapse[zz]
|
|
1657
|
+
alt_half = alt_half[zz]
|
|
1658
|
+
|
|
1659
|
+
# Iterate over the levels. If the lapse rate is < 2 K/km, check that it remains there over the next 2 kilometers
|
|
1660
|
+
for k, (gamma, alt) in enumerate(zip(lapse, alt_half)):
|
|
1661
|
+
if gamma < 2.0:
|
|
1662
|
+
step = 0.1
|
|
1663
|
+
test_alt = np.arange(alt, alt+2.0+step, step)
|
|
1664
|
+
test_lapse = np.interp(test_alt, alt_half, lapse)
|
|
1665
|
+
if np.all(test_lapse < 2.0):
|
|
1666
|
+
# Interpolate to the exact tropopause altitude where the lapse rate first crosses the 2 K/km
|
|
1667
|
+
# np.interp requires the x-coordinates to be sorted, hence the complicated formula to get k_inds
|
|
1668
|
+
k_inds = np.argsort(lapse[[k-1, k]]) + k - 1
|
|
1669
|
+
return np.interp(2.0, lapse[k_inds], alt_half[k_inds])
|
|
1670
|
+
|
|
1671
|
+
# If we get here, we failed to find the tropopause, so return a NaN or raise an error
|
|
1672
|
+
if raise_error:
|
|
1673
|
+
raise TropopauseError('Could not find a level meeting the WMO tropopause condition in the given profile')
|
|
1674
|
+
else:
|
|
1675
|
+
return np.nan
|
|
1676
|
+
|
|
1677
|
+
|
|
1678
|
+
def number_density_air(p, t):
|
|
1679
|
+
"""
|
|
1680
|
+
Calculate the ideal dry number density of air in molec. cm^-3
|
|
1681
|
+
|
|
1682
|
+
:param p: pressure in hPa
|
|
1683
|
+
:type p: float or :class:`numpy.ndarray`
|
|
1684
|
+
|
|
1685
|
+
:param t: temperature in K
|
|
1686
|
+
:type t: float or :class:`numpy.ndarray`
|
|
1687
|
+
|
|
1688
|
+
:return: ideal dry number density in molec. cm^-3
|
|
1689
|
+
:rtype: float or :class:`numpy.ndarray`
|
|
1690
|
+
"""
|
|
1691
|
+
R = const.gas_const # gas constant in cm^3 * hPa / (mol * K)
|
|
1692
|
+
return p / (R*t) * const.avogadro
|
|
1693
|
+
|
|
1694
|
+
|
|
1695
|
+
def effective_vertical_path(z, zmin, p=None, t=None, nair=None):
|
|
1696
|
+
"""
|
|
1697
|
+
Calculate the effective vertical path used by GFIT for a given z/P/T grid.
|
|
1698
|
+
|
|
1699
|
+
:param z: altitudes of the vertical levels. May be any unit, but note that the effective paths will be returned in
|
|
1700
|
+
the same unit.
|
|
1701
|
+
:type z: array-like
|
|
1702
|
+
|
|
1703
|
+
:param zmin: minimum altitude that the light ray reaches. This is given as ``zmin`` in the netCDF files and the .ray
|
|
1704
|
+
files. Must be in the same unit as ``z``.
|
|
1705
|
+
:type zmin: float
|
|
1706
|
+
|
|
1707
|
+
:param p: pressures of the vertical levels. Must be in hPa.
|
|
1708
|
+
:type p: array-like
|
|
1709
|
+
|
|
1710
|
+
:param t: temperatures of the vertical levels. Must be in K.
|
|
1711
|
+
:type t: array-like
|
|
1712
|
+
|
|
1713
|
+
:return: effective vertical paths in the same units as ``z``
|
|
1714
|
+
:rtype: array-like
|
|
1715
|
+
"""
|
|
1716
|
+
def integral(dz_in, lrp_in, sign):
|
|
1717
|
+
return dz_in * 0.5 * (1.0 + sign * lrp_in / 3 + lrp_in**2/12 + sign*lrp_in**3/60)
|
|
1718
|
+
|
|
1719
|
+
if nair is not None:
|
|
1720
|
+
d = nair
|
|
1721
|
+
elif p is not None and t is not None:
|
|
1722
|
+
d = number_density_air(p, t)
|
|
1723
|
+
else:
|
|
1724
|
+
raise TypeError('Either nair or p & t must be given')
|
|
1725
|
+
|
|
1726
|
+
vpath = np.zeros_like(d)
|
|
1727
|
+
|
|
1728
|
+
# From gfit/compute_vertical_paths.f, we need to find the first level above zmin
|
|
1729
|
+
# If there is no such level (which should not happen for TCCON), we treat the top
|
|
1730
|
+
# level this way
|
|
1731
|
+
try:
|
|
1732
|
+
klev = np.flatnonzero(z > zmin)[0]
|
|
1733
|
+
except IndexError:
|
|
1734
|
+
klev = np.size(z) - 1
|
|
1735
|
+
|
|
1736
|
+
# from gfit/compute_vertical_paths.f, the calculation for level i is
|
|
1737
|
+
# v_i = 0.5 * dz_{i+1} * (1 - l_{i+1}/3 + l_{i+1}**2/12 - l_{i+1}**3/60)
|
|
1738
|
+
# + 0.5 * dz_i * (1 + l_i/3 + l_i**2/12 + l_i**3/60)
|
|
1739
|
+
# where
|
|
1740
|
+
# dz_i = z_i - z_{i-1}
|
|
1741
|
+
# l_i = ln(d_{i-1}/d_i)
|
|
1742
|
+
# The top level has no i+1 term. This vector addition duplicates that calculation. The zeros padded to the beginning
|
|
1743
|
+
# and end of the difference vectors ensure that when there's no i+1 or i-1 term, it is given a value of 0.
|
|
1744
|
+
dz = np.concatenate([[0.0], np.diff(z[klev:]), [0.0]])
|
|
1745
|
+
log_rp = np.log(d[klev:-1] / d[klev+1:])
|
|
1746
|
+
log_rp = np.concatenate([[0.0], log_rp, [0.0]])
|
|
1747
|
+
|
|
1748
|
+
# The indexing is complicated here, but with how dz and log_rp are constructed, this makes sure that, for vpath[klev],
|
|
1749
|
+
# the first integral(...) term uses dz = z[klev+1] - z[klev] and log_rp = ln(d[klev]/d[klev+1]) and the second integral
|
|
1750
|
+
# term is 0 (as vpath[klev] needs to account for the surface location below). For all other terms, this combines the
|
|
1751
|
+
# contributions from the weight above and below each level, with different integration signs to account for how the
|
|
1752
|
+
# weights increase from the level below to the current level and decrease from the current level to the level above.
|
|
1753
|
+
vpath[klev:] = integral(dz[1:], log_rp[1:], sign=-1) + integral(dz[:-1], log_rp[:-1], sign=1)
|
|
1754
|
+
|
|
1755
|
+
# Now handle the surface - I don't fully understand how this is constructed mathematically, but the idea is that both
|
|
1756
|
+
# the levels in the prior above and below zmin need to contribute to the column, however that contribution needs to be
|
|
1757
|
+
# 0 below zmin.
|
|
1758
|
+
|
|
1759
|
+
dz = z[klev] - z[klev-1]
|
|
1760
|
+
xo = (zmin - z[klev-1])/dz
|
|
1761
|
+
log_rp = 0.0 if d[klev] <= 0 else np.log(d[klev-1]/d[klev])
|
|
1762
|
+
xl = log_rp * (1-xo)
|
|
1763
|
+
vpath[klev-1] += dz * (1-xo) * (1-xo-xl*(1+2*xo)/3 + (xl**2)*(1+3*xo)/12 + (xl**3)*(1+4*xo)/60)/2
|
|
1764
|
+
vpath[klev] += dz * (1-xo) * (1+xo+xl*(1+2*xo)/3 + (xl**2)*(1+3*xo)/12 - (xl**3)*(1+4*xo)/60)/2
|
|
1765
|
+
|
|
1766
|
+
return vpath
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
def get_ussa_for_alts(alts):
|
|
1770
|
+
"""
|
|
1771
|
+
Get temperature and pressure from the US standard atmosphere (USSA) for given altitudes.
|
|
1772
|
+
|
|
1773
|
+
Temperature is interpolated to the requested altitudes linearly, assuming that the lapse rate is constant between
|
|
1774
|
+
the levels defined by the USSA. Pressure is interpolated exponentially, i.e. ln(p) is interpolated linearly.
|
|
1775
|
+
|
|
1776
|
+
:param alts: altitudes, in kilometers, to calculate T and P for.
|
|
1777
|
+
:type alts: float or :class:`numpy.ndarray`
|
|
1778
|
+
|
|
1779
|
+
:return: temperatures (in K) and pressures (in hPa). Arrays will be the same shape as the input altitudes.
|
|
1780
|
+
:rtype: float or :class:`numpy.ndarray`, float or :class:`numpy.ndarray`
|
|
1781
|
+
"""
|
|
1782
|
+
# Need to interpolate pressure and temperature to the given altitudes. Will assume that temperature varies linearly
|
|
1783
|
+
# with altitude and pressure varies exponentially. Since p = p0 * exp(-z/H) then ln(p) = ln(p0) - z/H, therefore
|
|
1784
|
+
# we will linearly interpolate ln(p) w.r.t. altitude.
|
|
1785
|
+
z_coord = const.z_ussa
|
|
1786
|
+
t_coord = const.t_ussa
|
|
1787
|
+
p_coord = np.log(const.p_ussa)
|
|
1788
|
+
|
|
1789
|
+
interp_args = {'left': np.nan, 'right': np.nan}
|
|
1790
|
+
t = np.interp(alts, z_coord, t_coord, **interp_args)
|
|
1791
|
+
p = np.exp(np.interp(alts, z_coord, p_coord, **interp_args))
|
|
1792
|
+
|
|
1793
|
+
return t, p
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
def get_ussa_for_pres(pres):
|
|
1797
|
+
"""
|
|
1798
|
+
Get altitude and temperature from the US standard atmosphere (USSA) for given pressures.
|
|
1799
|
+
|
|
1800
|
+
Temperature is interpolated to the requested altitudes linearly, assuming that the lapse rate is constant between
|
|
1801
|
+
the levels defined by the USSA. Pressure is interpolated exponentially, i.e. ln(p) is interpolated linearly.
|
|
1802
|
+
|
|
1803
|
+
:param pres: pressures, in hPa, to calculate z and T for.
|
|
1804
|
+
:type pres: float or :class:`numpy.ndarray`
|
|
1805
|
+
|
|
1806
|
+
:return: temperatures (in K) and altitudes (in km). Arrays will be the same shape as the input altitudes.
|
|
1807
|
+
:rtype: float or :class:`numpy.ndarray`, float or :class:`numpy.ndarray`
|
|
1808
|
+
"""
|
|
1809
|
+
|
|
1810
|
+
# Since temperature varies linearly with altitude and altitude varies linearly vs. ln(p), interpolate both by the
|
|
1811
|
+
# log of pressure
|
|
1812
|
+
z_coord = np.flipud(const.z_ussa)
|
|
1813
|
+
t_coord = np.flipud(const.t_ussa)
|
|
1814
|
+
# must flip - np.interp expects its x-coordinates to be increasing.
|
|
1815
|
+
p_coord = np.flipud(np.log(const.p_ussa))
|
|
1816
|
+
|
|
1817
|
+
pres = np.log(pres)
|
|
1818
|
+
|
|
1819
|
+
interp_args = {'left': np.nan, 'right': np.nan}
|
|
1820
|
+
t = np.interp(pres, p_coord, t_coord, **interp_args)
|
|
1821
|
+
z = np.interp(pres, p_coord, z_coord, **interp_args)
|
|
1822
|
+
|
|
1823
|
+
return t, z
|
|
1824
|
+
|
|
1825
|
+
|
|
1826
|
+
def age_of_air(lat, z, ztrop, ref_lat=45.0):
|
|
1827
|
+
"""
|
|
1828
|
+
Calculate age of air using a function form from GGG 2014.
|
|
1829
|
+
|
|
1830
|
+
:param lat: the latitude(s) to calculate age of air for
|
|
1831
|
+
:type lat: float or :class:`numpy.ndarray`
|
|
1832
|
+
|
|
1833
|
+
:param z: the altitude(s) to calculate age of air for. If both ``z`` and ``lat`` given as a vectors, they must be
|
|
1834
|
+
the same shape. Must have units of kilometers.
|
|
1835
|
+
:type z: float or :class:`numpy.ndarray`
|
|
1836
|
+
|
|
1837
|
+
:param ztrop: the tropopause altitude, in kilometers.
|
|
1838
|
+
:type ztrop: float
|
|
1839
|
+
|
|
1840
|
+
:param ref_lat: the reference latitude for the cycle. This is where the exponential in latitude is maximized. 45N
|
|
1841
|
+
was chosen as the default as the center of the northern hemisphere, where most anthropogenic emissions are.
|
|
1842
|
+
:type ref_lat: float
|
|
1843
|
+
|
|
1844
|
+
:return: age of air, in years, as a numpy array
|
|
1845
|
+
:rtype: :class:`numpy.ndarray`
|
|
1846
|
+
"""
|
|
1847
|
+
# Force z to be a numpy array. This allows us to use numpy indexing for the extra (stratospheric) term below and
|
|
1848
|
+
# simultaneously ensures aoa is always a numpy array.
|
|
1849
|
+
if not isinstance(z, np.ndarray):
|
|
1850
|
+
z = np.array([z])
|
|
1851
|
+
|
|
1852
|
+
fl = lat/22.0
|
|
1853
|
+
aoa = 0.313 - 0.085 * np.exp(-((lat-ref_lat)/18)**2) - 0.268*np.exp(-1.42 * z / (z+ztrop)) * fl / np.sqrt(1+fl**2)
|
|
1854
|
+
|
|
1855
|
+
# We limit the calculation to z > ztrop here because that avoids a divide-by-0 warning
|
|
1856
|
+
# This term is really only kept in for completeness; in practice, it should never be used because we don't use
|
|
1857
|
+
# this term in the stratosphere.
|
|
1858
|
+
extra_term = 7.0 * (z[z > ztrop]-ztrop)/z[z > ztrop]
|
|
1859
|
+
aoa[z > ztrop] += extra_term
|
|
1860
|
+
return aoa
|
|
1861
|
+
|
|
1862
|
+
|
|
1863
|
+
def seasonal_cycle_factor(lat, z, ztrop, fyr, species, ref_lat=45.0):
|
|
1864
|
+
"""
|
|
1865
|
+
Calculate a factor to multiply a concentration by to account for the seasonal cycle.
|
|
1866
|
+
|
|
1867
|
+
:param lat: the latitude(s) to calculate age of air for
|
|
1868
|
+
:type lat: float or :class:`numpy.ndarray`
|
|
1869
|
+
|
|
1870
|
+
:param z: the altitude(s) to calculate age of air for. If both ``z`` and ``lat`` given as a vectors, they must be
|
|
1871
|
+
the same shape. Must have units of kilometers.
|
|
1872
|
+
:type z: float or :class:`numpy.ndarray`
|
|
1873
|
+
|
|
1874
|
+
:param ztrop: the tropopause altitude, in kilometers.
|
|
1875
|
+
:type ztrop: float
|
|
1876
|
+
|
|
1877
|
+
:param fyr: the fraction of the year that corresponds to this date. You can convert a date time to this value with
|
|
1878
|
+
:func:`date_to_frac_year`.
|
|
1879
|
+
:type fyr: float
|
|
1880
|
+
|
|
1881
|
+
:param species: a child class of :class:`~tccon_priors.TraceGasTropicsRecord` that defines a gas name and seasonal
|
|
1882
|
+
cycle coefficient. May be an instance of the class or the class itself. If the gas name is "co2", then a
|
|
1883
|
+
CO2-specific parameterization is used.
|
|
1884
|
+
:type species: :class:`~tccon_priors.TraceGasTropicsRecord`
|
|
1885
|
+
|
|
1886
|
+
:param ref_lat: reference latitude for the age of air. Set to 45N as an approximation of where the NH emissions are.
|
|
1887
|
+
:type ref_lat: float
|
|
1888
|
+
|
|
1889
|
+
:return: the seasonal cycle factor as a numpy array. Multiply this by a deseasonalized concentration at (lat, z) to
|
|
1890
|
+
get the concentration including the seasonal cycle
|
|
1891
|
+
"""
|
|
1892
|
+
if species.gas_seas_cyc_coeff is None:
|
|
1893
|
+
raise TypeError('The species record ({}) does not define a seasonal cycle coefficient')
|
|
1894
|
+
|
|
1895
|
+
aoa = age_of_air(lat, z, ztrop, ref_lat=ref_lat)
|
|
1896
|
+
if species.gas_name.lower() == 'co2':
|
|
1897
|
+
sv = np.sin(2*np.pi *(fyr - 0.834 - aoa))
|
|
1898
|
+
svnl = sv + 1.80 * np.exp(-((lat - 74)/41)**2)*(0.5 - sv**2)
|
|
1899
|
+
sca = svnl * np.exp(-aoa/0.20)*(1 + 1.33*np.exp(-((lat-76)/48)**2) * (z+6)/(z+1.4))
|
|
1900
|
+
else:
|
|
1901
|
+
sv = np.sin(2*np.pi * (fyr - 0.78)) # basic seasonal variation
|
|
1902
|
+
svl = sv * (lat / 15.0) / np.sqrt(1 + (lat / 15.0)**2.0) # latitude dependence
|
|
1903
|
+
sca = svl * np.exp(-aoa / 0.85) # altitude dependence
|
|
1904
|
+
|
|
1905
|
+
return 1 + sca * species.gas_seas_cyc_coeff
|
|
1906
|
+
|
|
1907
|
+
|
|
1908
|
+
def hf_ch4_slope_fit(yrs, a, b, c, t0):
|
|
1909
|
+
"""
|
|
1910
|
+
A fitting function appropriate to fit the trend of CH4 vs. HF slopes
|
|
1911
|
+
|
|
1912
|
+
This function has the form:
|
|
1913
|
+
|
|
1914
|
+
..math::
|
|
1915
|
+
a * exp(b*(t - t0)) + c
|
|
1916
|
+
|
|
1917
|
+
where t is given in years.
|
|
1918
|
+
|
|
1919
|
+
:param yrs: t in the above equation.
|
|
1920
|
+
:type yrs: :class:`numpy.ndarray`
|
|
1921
|
+
|
|
1922
|
+
:param a, b, c, t0: the fitting parameters in the above equation
|
|
1923
|
+
:type a, b, c, t0: float
|
|
1924
|
+
|
|
1925
|
+
:return: the predicted slopes at ``yrs``
|
|
1926
|
+
"""
|
|
1927
|
+
return a * np.exp(b*(yrs - t0)) + c
|
|
1928
|
+
|
|
1929
|
+
|
|
1930
|
+
# from https://stackoverflow.com/a/16562028
|
|
1931
|
+
def isoutlier(data, m=2):
|
|
1932
|
+
d = np.abs(data - np.nanmedian(data))
|
|
1933
|
+
mdev = np.nanmedian(d)
|
|
1934
|
+
s = d / mdev if mdev else np.zeros_like(data)
|
|
1935
|
+
|
|
1936
|
+
return s >= m
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def is_tropics(lat, doy, ages):
|
|
1940
|
+
return np.abs(lat) < 20.0
|
|
1941
|
+
|
|
1942
|
+
|
|
1943
|
+
def is_vortex(lat, doy, ages):
|
|
1944
|
+
if not isinstance(doy, np.ndarray):
|
|
1945
|
+
doy = np.full_like(lat, doy)
|
|
1946
|
+
xx_vortex = np.zeros_like(lat, dtype=np.bool_)
|
|
1947
|
+
xx_vortex[(doy > 140) & (doy < 245) & (lat < -55.0) & (ages > 3.25)] = True
|
|
1948
|
+
xx_vortex[(doy > 275) & (doy < 60) & (lat > 55.0) & (ages > 3.25)] = True
|
|
1949
|
+
return xx_vortex
|
|
1950
|
+
|
|
1951
|
+
|
|
1952
|
+
def is_midlat(lat, doy, ages):
|
|
1953
|
+
return ~is_tropics(lat, doy, ages) & ~is_vortex(lat, doy, ages)
|
|
1954
|
+
|
|
1955
|
+
|
|
1956
|
+
def is_overworld(potential_temp, pressure, trop_pres):
|
|
1957
|
+
return (potential_temp >= 380) & (pressure <= trop_pres)
|
|
1958
|
+
|
|
1959
|
+
|
|
1960
|
+
def npdate_to_datetime(numpy_date):
|
|
1961
|
+
numpy_date = numpy_date.astype('datetime64[s]')
|
|
1962
|
+
ts = (numpy_date - np.datetime64('1970-01-01T00:00:00Z', 's')) / np.timedelta64(1, 's')
|
|
1963
|
+
return dt.datetime.utcfromtimestamp(ts)
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def date_to_decimal_year(date_in):
|
|
1967
|
+
"""
|
|
1968
|
+
Convert a datetime object to a decimal year.
|
|
1969
|
+
|
|
1970
|
+
A decimal year is e.g. 2018.5, where the part before the decimal is the year itself and after the decimal is how
|
|
1971
|
+
much of the year has passed.
|
|
1972
|
+
|
|
1973
|
+
:param date_in: the datetime object to convert
|
|
1974
|
+
:type date_in: datetime-like
|
|
1975
|
+
|
|
1976
|
+
:return: the decimal year
|
|
1977
|
+
:rtype: float
|
|
1978
|
+
"""
|
|
1979
|
+
if date_in is not None:
|
|
1980
|
+
return date_in.year + date_to_frac_year(date_in)
|
|
1981
|
+
else:
|
|
1982
|
+
return np.nan
|
|
1983
|
+
|
|
1984
|
+
|
|
1985
|
+
def day_of_year(date_in):
|
|
1986
|
+
return clams_day_of_year(date_in) - 1
|
|
1987
|
+
|
|
1988
|
+
|
|
1989
|
+
def clams_day_of_year(date_in):
|
|
1990
|
+
return float(date_in.strftime('%j'))
|
|
1991
|
+
|
|
1992
|
+
|
|
1993
|
+
def date_to_frac_year(date_in):
|
|
1994
|
+
"""
|
|
1995
|
+
Convert a datetime object to a fraction of a year.
|
|
1996
|
+
|
|
1997
|
+
The fraction is essentially how much of the year has passed, so June 1st (of any year) becomes ~0.416.
|
|
1998
|
+
|
|
1999
|
+
Note: this function assumes 365.25 days per year. This is an imperfect solution, ideally a fractional year should
|
|
2000
|
+
really describe the fraction of an orbit the Earth has completed, so only use this for things where +/- a day is an
|
|
2001
|
+
insignificant error.
|
|
2002
|
+
|
|
2003
|
+
:param date_in: the datetime object to convert
|
|
2004
|
+
:type date_in: datetime-like
|
|
2005
|
+
|
|
2006
|
+
:return: the fractional year
|
|
2007
|
+
:rtype: float
|
|
2008
|
+
"""
|
|
2009
|
+
return day_of_year(date_in) / days_per_year # since there's about and extra quarter of a day per year that gives us leap years
|
|
2010
|
+
|
|
2011
|
+
|
|
2012
|
+
def frac_year_to_doy(yr_in):
|
|
2013
|
+
"""
|
|
2014
|
+
Convert a fractional year to a day of year.
|
|
2015
|
+
|
|
2016
|
+
Internally, this multiplies by 365.25, so see the warning in the docstring for :func:`date_to_frac_year` about
|
|
2017
|
+
its precision. Crucially, this is NOT a reliable inverse operation to :func:`date_to_frac_year`.
|
|
2018
|
+
|
|
2019
|
+
:param yr_in: the fractional year to convert
|
|
2020
|
+
:type yr_in: float
|
|
2021
|
+
|
|
2022
|
+
:return: the number of days since 1 Jan
|
|
2023
|
+
:rtype: float
|
|
2024
|
+
"""
|
|
2025
|
+
return yr_in * days_per_year
|
|
2026
|
+
|
|
2027
|
+
|
|
2028
|
+
def frac_years_to_reldelta(frac_year, allow_nans=True):
|
|
2029
|
+
"""
|
|
2030
|
+
Convert a fractional year to a :class:`relativedelta` from dateutils.
|
|
2031
|
+
|
|
2032
|
+
Note: like the other fraction/decimal year functions, this assumes 365.25 days/year internally. Therefore, this
|
|
2033
|
+
should function correctly as an inverse operation to date_to_frac_year when added back to Jan 1 of the year in
|
|
2034
|
+
question.
|
|
2035
|
+
|
|
2036
|
+
:param frac_year: the fractional year(s) (e.g 2.5 for 2 and a half years) to convert
|
|
2037
|
+
:type frac_year: float or a collection of floats
|
|
2038
|
+
|
|
2039
|
+
:param allow_nans: whether to permit NaNs in the decimal years. If ``True``, then NaNs will be retained in the
|
|
2040
|
+
output list. If ``False``, an error is raised if NaNs are found in ``dec_year``.
|
|
2041
|
+
:type allow_nans: bool
|
|
2042
|
+
|
|
2043
|
+
:return: a list of dateutils :class:`relativedelta` objects or a single :class:`relativedelta` if a scalar
|
|
2044
|
+
``frac_year`` was given.
|
|
2045
|
+
:rtype: :class:`relativedelta` or list(:class:`relativedelta`)
|
|
2046
|
+
"""
|
|
2047
|
+
if isinstance(frac_year, float):
|
|
2048
|
+
return_scalar = True
|
|
2049
|
+
frac_year = [frac_year]
|
|
2050
|
+
else:
|
|
2051
|
+
return_scalar = False
|
|
2052
|
+
if not allow_nans and np.any(np.isnan(frac_year)):
|
|
2053
|
+
raise ValueError('NaNs not permitted in frac_year. Either remove them, or set `allow_nans=True`')
|
|
2054
|
+
age_years = np.floor(frac_year)
|
|
2055
|
+
age_fracs = np.mod(frac_year, 1)
|
|
2056
|
+
rdels = [relativedelta(years=y, days=days_per_year * d) if not (np.isnan(y) or np.isnan(d)) else np.nan for y, d in zip(age_years, age_fracs)]
|
|
2057
|
+
if return_scalar:
|
|
2058
|
+
rdels = rdels[0]
|
|
2059
|
+
|
|
2060
|
+
return rdels
|
|
2061
|
+
|
|
2062
|
+
|
|
2063
|
+
def timedelta_to_frac_year(timedelta):
|
|
2064
|
+
"""
|
|
2065
|
+
Convert a concrete timedelta to fractional years
|
|
2066
|
+
|
|
2067
|
+
:param timedelta: the timedelta to convert
|
|
2068
|
+
:type timedelta: :class:`datetime.timedelta`
|
|
2069
|
+
|
|
2070
|
+
:return: the time delta as a fraction of years, assuming 365.25 days per year.
|
|
2071
|
+
:rtype: float
|
|
2072
|
+
"""
|
|
2073
|
+
|
|
2074
|
+
return timedelta.total_seconds() / (days_per_year * 24 * 3600)
|
|
2075
|
+
|
|
2076
|
+
|
|
2077
|
+
def decimal_year_to_date(dec_year, date_type=dt.datetime):
|
|
2078
|
+
"""
|
|
2079
|
+
Convert decimal year or years (e.g. 2018.5) to a datetime-like object
|
|
2080
|
+
|
|
2081
|
+
:param dec_year: the decimal year or years to convert. May be any kind of collection (if passing multiple values) so
|
|
2082
|
+
long as it supports iteration and that iteration returns scalar values (i.e. a 2D numpy array will not work because
|
|
2083
|
+
iteration returns rows).
|
|
2084
|
+
|
|
2085
|
+
:param date_type: what type to convert the decimal years in to. May be any time that can be called
|
|
2086
|
+
``date_type(year, month, day)``.
|
|
2087
|
+
|
|
2088
|
+
:return: the converted dates in the type ``date_type``. If a single decimal date was passed in, then a single value
|
|
2089
|
+
of type ``date_type`` is returned. If a collection was passed in, then the dates will be returned in a 1D numpy
|
|
2090
|
+
array.
|
|
2091
|
+
"""
|
|
2092
|
+
|
|
2093
|
+
if np.any(np.isnan(dec_year)):
|
|
2094
|
+
raise NotImplementedError('NaNs in the input decimal years not implemented')
|
|
2095
|
+
|
|
2096
|
+
try:
|
|
2097
|
+
dec_year[0]
|
|
2098
|
+
except (TypeError, IndexError):
|
|
2099
|
+
dec_year = [dec_year]
|
|
2100
|
+
return_as_scalar = True
|
|
2101
|
+
else:
|
|
2102
|
+
return_as_scalar = False
|
|
2103
|
+
|
|
2104
|
+
years = np.array([int(d) for d in dec_year])
|
|
2105
|
+
frac_yrs = np.array([d % 1 for d in dec_year])
|
|
2106
|
+
dates = np.array([date_type(y, 1, 1) + frac_years_to_reldelta(fr, allow_nans=False) for y, fr in zip(years, frac_yrs)])
|
|
2107
|
+
|
|
2108
|
+
if return_as_scalar:
|
|
2109
|
+
return dates[0]
|
|
2110
|
+
else:
|
|
2111
|
+
return dates
|
|
2112
|
+
|
|
2113
|
+
|
|
2114
|
+
def start_of_month(date_in, out_type=dt.date):
|
|
2115
|
+
"""
|
|
2116
|
+
Get a date-like object corresponding to the beginning of the month of ``date_in``
|
|
2117
|
+
|
|
2118
|
+
:param date_in: Any date-like object that has attributes ``year`` and ``month``
|
|
2119
|
+
:param out_type: A type whose constructor accepts the keyword arguments ``year``, ``month``, and ``day``.
|
|
2120
|
+
:return: an instance of ``out_type`` set to day 1, 00:00:00 of the month of ``date_in``.
|
|
2121
|
+
"""
|
|
2122
|
+
return out_type(year=date_in.year, month=date_in.month, day=1)
|
|
2123
|
+
|
|
2124
|
+
|
|
2125
|
+
def relativedelta2string(rdelta):
|
|
2126
|
+
parts = ('years', 'months', 'days', 'hours', 'minutes', 'seconds')
|
|
2127
|
+
time_parts = []
|
|
2128
|
+
for p in parts:
|
|
2129
|
+
tp = getattr(rdelta, p)
|
|
2130
|
+
if tp > 0:
|
|
2131
|
+
time_parts.append('{} {}'.format(tp, p))
|
|
2132
|
+
|
|
2133
|
+
return ', '.join(time_parts)
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
def gravity(gdlat,altit):
|
|
2137
|
+
"""
|
|
2138
|
+
copy/pasted from fortran routine comments
|
|
2139
|
+
This is used to convert
|
|
2140
|
+
|
|
2141
|
+
Input Parameters:
|
|
2142
|
+
gdlat GeoDetric Latitude (degrees)
|
|
2143
|
+
altit Geometric Altitude (km)
|
|
2144
|
+
|
|
2145
|
+
Output Parameter:
|
|
2146
|
+
gravity Effective Gravitational Acceleration (m/s2)
|
|
2147
|
+
radius Radius of earth at gdlat
|
|
2148
|
+
|
|
2149
|
+
Computes the effective Earth gravity at a given latitude and altitude.
|
|
2150
|
+
This is the sum of the gravitational and centripital accelerations.
|
|
2151
|
+
These are based on equation I.2.4-(17) in US Standard Atmosphere 1962
|
|
2152
|
+
The Earth is assumed to be an oblate ellipsoid, with a ratio of the
|
|
2153
|
+
major to minor axes = sqrt(1+con) where con=.006738
|
|
2154
|
+
This eccentricity makes the Earth's gravititational field smaller at
|
|
2155
|
+
the poles and larger at the equator than if the Earth were a sphere
|
|
2156
|
+
of the same mass. [At the equator, more of the mass is directly
|
|
2157
|
+
below, whereas at the poles more is off to the sides). This effect
|
|
2158
|
+
also makes the local mid-latitude gravity field not point towards
|
|
2159
|
+
the center of mass.
|
|
2160
|
+
|
|
2161
|
+
The equation used in this subroutine agrees with the International
|
|
2162
|
+
Gravitational Formula of 1967 (Helmert's equation) within 0.005%.
|
|
2163
|
+
|
|
2164
|
+
Interestingly, since the centripital effect of the Earth's rotation
|
|
2165
|
+
(-ve at equator, 0 at poles) has almost the opposite shape to the
|
|
2166
|
+
second order gravitational field (+ve at equator, -ve at poles),
|
|
2167
|
+
their sum is almost constant so that the surface gravity could be
|
|
2168
|
+
approximated (.07%) by the simple expression g=0.99746*GM/radius^2,
|
|
2169
|
+
the latitude variation coming entirely from the variation of surface
|
|
2170
|
+
r with latitude. This simple equation is not used in this subroutine.
|
|
2171
|
+
"""
|
|
2172
|
+
|
|
2173
|
+
d2r=3.14159265/180.0 # Conversion from degrees to radians
|
|
2174
|
+
gm=3.9862216e+14 # Gravitational constant times Earth's Mass (m3/s2)
|
|
2175
|
+
omega=7.292116E-05 # Earth's angular rotational velocity (radians/s)
|
|
2176
|
+
con=0.006738 # (a/b)**2-1 where a & b are equatorial & polar radii
|
|
2177
|
+
shc=1.6235e-03 # 2nd harmonic coefficient of Earth's gravity field
|
|
2178
|
+
eqrad=6378178.0 # Equatorial Radius (meters)
|
|
2179
|
+
|
|
2180
|
+
gclat=arctan(tan(d2r*gdlat)/(1.0+con)) # radians
|
|
2181
|
+
|
|
2182
|
+
radius=1000.0*altit+eqrad/np.sqrt(1.0+con*sin(gclat)**2)
|
|
2183
|
+
ff=(radius/eqrad)**2
|
|
2184
|
+
hh=radius*omega**2
|
|
2185
|
+
ge=gm/eqrad**2 # = gravity at Re
|
|
2186
|
+
|
|
2187
|
+
gravity=(ge*(1-shc*(3.0*sin(gclat)**2-1)/ff)/ff-hh*cos(gclat)**2)*(1+0.5*(sin(gclat)*cos(gclat)*(hh/ge+2.0*shc/ff**2))**2)
|
|
2188
|
+
|
|
2189
|
+
return gravity, radius
|
|
2190
|
+
|
|
2191
|
+
|
|
2192
|
+
def geopotential_height_to_altitude(gph, lat, alt):
|
|
2193
|
+
"""
|
|
2194
|
+
Convert a geopotential height in m^2 s^-2 to meters
|
|
2195
|
+
|
|
2196
|
+
:param gph: geopotential height in m^2 s^-2.
|
|
2197
|
+
:type gph: float
|
|
2198
|
+
|
|
2199
|
+
:param lat: geographic latitude (in degrees, south is negative)
|
|
2200
|
+
:type lat: float
|
|
2201
|
+
|
|
2202
|
+
:param alt: altitude of the TCCON site in kilometers. If set to 0, will use gravity at the surface for the given
|
|
2203
|
+
latitude.
|
|
2204
|
+
:type alt: float
|
|
2205
|
+
|
|
2206
|
+
:return: the geopotential height converted to meters
|
|
2207
|
+
"""
|
|
2208
|
+
gravity_at_site, _ = gravity(lat, alt)
|
|
2209
|
+
return gph / gravity_at_site
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
def to_unix_time(datetime):
|
|
2213
|
+
"""
|
|
2214
|
+
Convert a datetime-like object into Unix time (seconds since midnight, 1 Jan 1970)
|
|
2215
|
+
|
|
2216
|
+
:param datetime: the datetime to convert. May be any type that can have a :class:`datetime.datetime` object
|
|
2217
|
+
subtracted from it, and for which the subtraction has a method `total_seconds` that returns the time delta as
|
|
2218
|
+
a number of seconds. Both :class:`datetime.datetime` and :class:`pandas.Timestamp` are examples.
|
|
2219
|
+
|
|
2220
|
+
:return: unix time
|
|
2221
|
+
:rtype: float
|
|
2222
|
+
"""
|
|
2223
|
+
return (datetime - dt.datetime(1970, 1, 1)).total_seconds()
|
|
2224
|
+
|
|
2225
|
+
|
|
2226
|
+
def from_unix_time(utime, out_type=dt.datetime):
|
|
2227
|
+
"""
|
|
2228
|
+
Convert a unix time into a datetime object.
|
|
2229
|
+
|
|
2230
|
+
:param utime: the unix time (seconds since midnight, 1 Jan 1970)
|
|
2231
|
+
:type utime: float
|
|
2232
|
+
|
|
2233
|
+
:param out_type: optional, a type that represents a datetime which has an init method such that
|
|
2234
|
+
``out_type(year, month, day)`` returns a object representing that time and can be added with a
|
|
2235
|
+
:class:`datetime.timedelta``.
|
|
2236
|
+
:type out_type: type
|
|
2237
|
+
|
|
2238
|
+
:return: a datetime object of the type specified by ``out_type``.
|
|
2239
|
+
"""
|
|
2240
|
+
return out_type(1970, 1, 1) + dt.timedelta(seconds=utime)
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
def mod_file_name(prefix,date,time_step,site_lat,site_lon_180,ew,ns,mod_path,round_latlon=True,in_utc=True):
|
|
2244
|
+
|
|
2245
|
+
YYYYMMDD = date.strftime('%Y%m%d')
|
|
2246
|
+
HH = date.strftime('%H')
|
|
2247
|
+
if in_utc:
|
|
2248
|
+
HH += 'Z'
|
|
2249
|
+
else:
|
|
2250
|
+
HH += 'L'
|
|
2251
|
+
|
|
2252
|
+
if round_latlon:
|
|
2253
|
+
site_lat = round(abs(site_lat))
|
|
2254
|
+
site_lon = round(abs(site_lon_180))
|
|
2255
|
+
latlon_precision = 0
|
|
2256
|
+
else:
|
|
2257
|
+
site_lat = abs(site_lat)
|
|
2258
|
+
site_lon = abs(site_lon_180)
|
|
2259
|
+
latlon_precision = 2
|
|
2260
|
+
if time_step < timedelta(days=1):
|
|
2261
|
+
mod_fmt = '{{prefix}}_{{ymd}}{{hh}}_{{lat:0>2.{prec}f}}{{ns:>1}}_{{lon:0>3.{prec}f}}{{ew:>1}}.mod'.format(prec=latlon_precision)
|
|
2262
|
+
else:
|
|
2263
|
+
mod_fmt = '{{prefix}}_{{ymd}}_{{lat:0>2.{prec}f}}{{ns:>1}}_{{lon:0>3.{prec}f}}{{ew:>1}}.mod'.format(prec=latlon_precision)
|
|
2264
|
+
|
|
2265
|
+
mod_name = mod_fmt.format(prefix=prefix, ymd=YYYYMMDD, hh=HH, lat=site_lat, ns=ns, lon=site_lon, ew=ew)
|
|
2266
|
+
return mod_name
|
|
2267
|
+
|
|
2268
|
+
|
|
2269
|
+
def mod_file_name_for_priors(datetime, site_lat, site_lon_180, prefix='FPIT', **kwargs):
|
|
2270
|
+
if site_lon_180 > 180:
|
|
2271
|
+
site_lon_180 -= 360
|
|
2272
|
+
ew = format_lon(site_lon_180)[-1]
|
|
2273
|
+
ns = format_lat(site_lat)[-1]
|
|
2274
|
+
return mod_file_name(prefix=prefix, date=datetime, time_step=dt.timedelta(hours=3), site_lat=site_lat,
|
|
2275
|
+
site_lon_180=site_lon_180, ew=ew, ns=ns, mod_path='', **kwargs)
|
|
2276
|
+
|
|
2277
|
+
|
|
2278
|
+
def parse_date_range(datestr):
|
|
2279
|
+
def parse_date(datestr):
|
|
2280
|
+
try:
|
|
2281
|
+
date_out = dt.datetime.strptime(datestr, '%Y%m%d')
|
|
2282
|
+
except ValueError:
|
|
2283
|
+
date_out = dt.datetime.strptime(datestr, '%Y%m%d_%H')
|
|
2284
|
+
return date_out
|
|
2285
|
+
|
|
2286
|
+
dates = datestr.split('-')
|
|
2287
|
+
start_date = parse_date(dates[0])
|
|
2288
|
+
if len(dates) > 1:
|
|
2289
|
+
end_date = parse_date(dates[1])
|
|
2290
|
+
else:
|
|
2291
|
+
end_date = start_date + timedelta(days=1)
|
|
2292
|
+
|
|
2293
|
+
return start_date, end_date
|
|
2294
|
+
|
|
2295
|
+
|
|
2296
|
+
def get_ggg_path(subdir, subdir_name):
|
|
2297
|
+
gggpath = os.getenv('GGGPATH')
|
|
2298
|
+
if gggpath is None:
|
|
2299
|
+
raise GGGPathError('Could not find the GGGPATH environmental variable. Please specify an explicit {}.'.format(subdir_name))
|
|
2300
|
+
full_subdir = os.path.join(gggpath, subdir)
|
|
2301
|
+
if not os.path.isdir(full_subdir):
|
|
2302
|
+
raise GGGPathError('Could not find default {} {}'.format(subdir_name, full_subdir))
|
|
2303
|
+
|
|
2304
|
+
return full_subdir
|
|
2305
|
+
|
|
2306
|
+
|
|
2307
|
+
def check_site_lat_lon_alt(abbrev, lat=None, lon=None, alt=None):
|
|
2308
|
+
"""
|
|
2309
|
+
Verify that the input site abbreviation, latitude, longitude, and altitude are consistent.
|
|
2310
|
+
|
|
2311
|
+
This checks that (1) if any of lat/lon/alt are given, all are, (2) if any inputs are > 1 long all those that are
|
|
2312
|
+
are the same length, (3) if any inputs are scalar (None, int, float, or string) they are converted to tuples
|
|
2313
|
+
of the proper length, and (4) all longitudes are given in [0, 360)
|
|
2314
|
+
|
|
2315
|
+
:param abbrev: site abbreviation(s), as string or iterable of such
|
|
2316
|
+
:type abbrev: str or list(str)
|
|
2317
|
+
|
|
2318
|
+
:param lat: site latitude(s), as float or iterable of such, or None
|
|
2319
|
+
:type lat: float or list(float)
|
|
2320
|
+
|
|
2321
|
+
:param lon: site longitude(s), as float or iterable of such, or None
|
|
2322
|
+
:type lon: float or list(float)
|
|
2323
|
+
|
|
2324
|
+
:param alt: site altitude(s), as float or iterable of such, or None
|
|
2325
|
+
:type alt: float or list(float)
|
|
2326
|
+
|
|
2327
|
+
:return: abbrev, lat, lon, and "alt" converted to tuples of the same length. Scalar values are repeated.
|
|
2328
|
+
:rtype: tuples
|
|
2329
|
+
"""
|
|
2330
|
+
def to_tuple(val):
|
|
2331
|
+
if isinstance(val, (int, float, str, type(None))):
|
|
2332
|
+
val = (val,)
|
|
2333
|
+
return val, len(val)
|
|
2334
|
+
|
|
2335
|
+
are_none = [x is None for x in (lat, lon, alt)]
|
|
2336
|
+
abb_tuple, _ = to_tuple(abbrev)
|
|
2337
|
+
if all(are_none):
|
|
2338
|
+
if len(abb_tuple) == 1 and abb_tuple[0] == 'all':
|
|
2339
|
+
pass
|
|
2340
|
+
elif any(abb not in site_dict for abb in abb_tuple):
|
|
2341
|
+
raise ValueError('One or more site abbreviation not a predefined TCCON site, must give lat, lon, alt')
|
|
2342
|
+
elif any(are_none):
|
|
2343
|
+
if len(abb_tuple) == 1 and abb_tuple[0] == 'all':
|
|
2344
|
+
raise ValueError('Running all sites is incompatible with giving custom lat/lons')
|
|
2345
|
+
else:
|
|
2346
|
+
raise ValueError('If any of lat, lon, or alt are given, all must be given')
|
|
2347
|
+
|
|
2348
|
+
vals = dict()
|
|
2349
|
+
n = dict()
|
|
2350
|
+
vals['abbrev'], n['abbrev'] = to_tuple(abbrev)
|
|
2351
|
+
vals['lat'], n['lat'] = to_tuple(lat)
|
|
2352
|
+
vals['lon'], n['lon'] = to_tuple(lon)
|
|
2353
|
+
vals['alt'], n['alt'] = to_tuple(alt)
|
|
2354
|
+
|
|
2355
|
+
max_n = max(v for v in n.values())
|
|
2356
|
+
for k in vals.keys():
|
|
2357
|
+
if n[k] == 1:
|
|
2358
|
+
vals[k] *= max_n
|
|
2359
|
+
elif n[k] != max_n:
|
|
2360
|
+
raise ValueError('{} is not a single value or the same length as the longest input'.format(k))
|
|
2361
|
+
|
|
2362
|
+
# Constrain longitude between 0 and 360
|
|
2363
|
+
vals['lon'] = tuple([l + 360 if l is not None and l < 0 else l for l in vals['lon']])
|
|
2364
|
+
if any([l < 0 or l >= 360 for l in vals['lon'] if l is not None]):
|
|
2365
|
+
raise ValueError('Longitude outside of valid range (-360, +360, exclusive) found')
|
|
2366
|
+
|
|
2367
|
+
return vals['abbrev'], vals['lat'], vals['lon'], vals['alt']
|
|
2368
|
+
|
|
2369
|
+
|
|
2370
|
+
def dry2wet(dmf, h2o_dmf):
|
|
2371
|
+
return dmf / (1 + h2o_dmf)
|
|
2372
|
+
|
|
2373
|
+
|
|
2374
|
+
def interp_to_zgrid(profile_dict, zgrid, gas_extrap_method='linear'):
|
|
2375
|
+
"""
|
|
2376
|
+
Interpolate the output profile variables to a desired altitude grid.
|
|
2377
|
+
|
|
2378
|
+
This mimics the behavior in `read_refvmrs` of the Fortran gsetup code. It interpolates the output variables linearly
|
|
2379
|
+
with respect to altitude. It will extrapolate linearly towards the surface, but will raise an error if the profile
|
|
2380
|
+
does not go high enough.
|
|
2381
|
+
|
|
2382
|
+
:param profile_dict: a dictionary of profile variables. Must include the altitude levels as the key "Height". The
|
|
2383
|
+
keys "gas_record_dates" and "gas_date" are treated as dates for interpolation.
|
|
2384
|
+
:type profile_dict: dict
|
|
2385
|
+
|
|
2386
|
+
:param zgrid: any specification of a fixed altitude grid accepted by :func:`_setup_zgrid`, i.e. ``None`` to do no
|
|
2387
|
+
interpolation, a path to an integral file, or an array directly specifying the altitude grid.
|
|
2388
|
+
:type zgrid: None, str, or :class:`numpy.ndarray`.
|
|
2389
|
+
|
|
2390
|
+
:return: the profile dictionary with its variables interpolated/extrapolated to the new altitude grid. It will
|
|
2391
|
+
also have been modified in-place.
|
|
2392
|
+
:rtype: dict
|
|
2393
|
+
"""
|
|
2394
|
+
# Specify pairs of functions to convert and unconvert the values of the input arrays. Each key must match a key in
|
|
2395
|
+
# the profile_dict, and the values must be two element tuples, where the first element is the function to use to
|
|
2396
|
+
# convert the values in the dict to ones that can be interpolated and the second does the reverse process.
|
|
2397
|
+
met_variables = ('Height', 'Temp', 'Pressure', 'PT', 'EqL')
|
|
2398
|
+
dt_converters = (_datetime2float, _float2datetime)
|
|
2399
|
+
converters = {'gas_record_dates': dt_converters,
|
|
2400
|
+
'gas_date': dt_converters}
|
|
2401
|
+
|
|
2402
|
+
zgrid = _setup_zgrid(zgrid)
|
|
2403
|
+
if zgrid is None:
|
|
2404
|
+
return profile_dict
|
|
2405
|
+
|
|
2406
|
+
profile_z = profile_dict['Height'].copy()
|
|
2407
|
+
for k, v in profile_dict.items():
|
|
2408
|
+
if k in converters:
|
|
2409
|
+
v = converters[k][0](v)
|
|
2410
|
+
|
|
2411
|
+
v = xr.DataArray(v, coords=[profile_z], dims=['z'])
|
|
2412
|
+
if gas_extrap_method == 'linear' or k in met_variables:
|
|
2413
|
+
# we always want to linearly extrapolate the met variables, especially since they often act as vertical
|
|
2414
|
+
# coordinates. If more met variables are added, you'll have to update the list.
|
|
2415
|
+
fills = 'extrapolate'
|
|
2416
|
+
elif gas_extrap_method in ('const', 'constant'):
|
|
2417
|
+
fills = (v[0], v[-1])
|
|
2418
|
+
else:
|
|
2419
|
+
raise ValueError('extrap_method must be one of: "linear", "const", or "constant".')
|
|
2420
|
+
v = v.interp(z=zgrid, method='linear', kwargs={'fill_value': fills})
|
|
2421
|
+
|
|
2422
|
+
if k in converters:
|
|
2423
|
+
v = converters[k][1](v.data)
|
|
2424
|
+
else:
|
|
2425
|
+
v = v.data
|
|
2426
|
+
profile_dict[k] = v
|
|
2427
|
+
|
|
2428
|
+
return profile_dict
|
|
2429
|
+
|
|
2430
|
+
|
|
2431
|
+
def _setup_zgrid(zgrid):
|
|
2432
|
+
"""
|
|
2433
|
+
Setup a fixed altitude grid
|
|
2434
|
+
:param zgrid:
|
|
2435
|
+
:return:
|
|
2436
|
+
"""
|
|
2437
|
+
if isinstance(zgrid, str):
|
|
2438
|
+
int_info = _read_integral_file(zgrid)
|
|
2439
|
+
return int_info['Height']
|
|
2440
|
+
elif not isinstance(zgrid, (type(None), np.ndarray, xr.DataArray)):
|
|
2441
|
+
raise TypeError('zgrid must be a string, None, numpy array, or xarray DataArray')
|
|
2442
|
+
else:
|
|
2443
|
+
return zgrid
|
|
2444
|
+
|
|
2445
|
+
|
|
2446
|
+
def _read_integral_file(integral_file, as_dataframes=False):
|
|
2447
|
+
"""
|
|
2448
|
+
Read an integral file that defines an altitude grid for GGG
|
|
2449
|
+
|
|
2450
|
+
:param integral_file: the path to the integral file
|
|
2451
|
+
:type integral_file: str
|
|
2452
|
+
|
|
2453
|
+
:param as_dataframes: if ``True``, the information in the file is returned as a single dataframe. If ``False``, it
|
|
2454
|
+
is returned as a dict of numpy arrays.
|
|
2455
|
+
:type as_dataframes: bool
|
|
2456
|
+
|
|
2457
|
+
:return: the table of altitudes and mean molecular weights.
|
|
2458
|
+
:rtype: :class:`pandas.DataFrame` or dict
|
|
2459
|
+
"""
|
|
2460
|
+
df = pd.read_csv(integral_file, sep=r'\s+', header=None, names=['Height', 'mmw'])
|
|
2461
|
+
if as_dataframes:
|
|
2462
|
+
return df
|
|
2463
|
+
else:
|
|
2464
|
+
return {k: v.to_numpy() for k, v in df.items()}
|
|
2465
|
+
|
|
2466
|
+
|
|
2467
|
+
def _datetime2float(dtarray):
|
|
2468
|
+
return np.array([np.nan if d is None else to_unix_time(d) for d in dtarray])
|
|
2469
|
+
|
|
2470
|
+
|
|
2471
|
+
def _float2datetime(dtarray):
|
|
2472
|
+
return np.array([None if np.isnan(d) else from_unix_time(d, pd.Timestamp) for d in dtarray])
|
|
2473
|
+
|
|
2474
|
+
|
|
2475
|
+
def frac_ydh_to_date(year, day, hour):
|
|
2476
|
+
"""
|
|
2477
|
+
Convert a fractional year, day, and hour from a GGG output file to a date
|
|
2478
|
+
|
|
2479
|
+
Some GGG output files record year as :math:`year + (day + hour/24)/366`
|
|
2480
|
+
and day as :math:`day + hour/24`. This function converts such values to
|
|
2481
|
+
a Pandas DatetimeIndex.
|
|
2482
|
+
|
|
2483
|
+
:param year: array of years, as decimal numbers
|
|
2484
|
+
:type year: array(float)
|
|
2485
|
+
|
|
2486
|
+
:param day: array of days-of-year, with Jan 1 = 1, as decimal numbers.
|
|
2487
|
+
:type day: array(float)
|
|
2488
|
+
|
|
2489
|
+
:param hour: array of hours of day.
|
|
2490
|
+
:type hour: array(float)
|
|
2491
|
+
|
|
2492
|
+
:return: array of datetimes
|
|
2493
|
+
:rtype: :class:`pandas.DatetimeIndex`
|
|
2494
|
+
"""
|
|
2495
|
+
# In many output files, the year and day are calculated to represent the
|
|
2496
|
+
# whole date, i.e. year is a decimal number where the decimal part is the
|
|
2497
|
+
# day + hour in fraction of a year. Same for day. Normally we could just
|
|
2498
|
+
# convert them to an integer and be fine, except there are cases where the
|
|
2499
|
+
# day rolls over into the next day, e.g. because we're working on a western
|
|
2500
|
+
# hemisphere site that starts measuring in one UTC day and keeps measuring
|
|
2501
|
+
# through to the next, so the hour exceeds 24.
|
|
2502
|
+
#
|
|
2503
|
+
# We account for this by subtracting off the fractional part of the year and
|
|
2504
|
+
# day computed from the smaller elements. However, we don't subtract the
|
|
2505
|
+
# whole amount off, because we don't want floating point uncertainty to
|
|
2506
|
+
# take us to less than the correct integer. E.g. if the day is 60.5 and
|
|
2507
|
+
# we compute hour/24 = 0.50000000001, then int(day - hour/24) = 59.
|
|
2508
|
+
|
|
2509
|
+
# correct the year
|
|
2510
|
+
year_frac = day / 366
|
|
2511
|
+
year = (year - 0.99*year_frac).astype('int')
|
|
2512
|
+
|
|
2513
|
+
# correct the day
|
|
2514
|
+
day_frac = hour / 24
|
|
2515
|
+
day = (day - 0.99*day_frac).astype('int')
|
|
2516
|
+
|
|
2517
|
+
return ydh_to_date(year, day, hour)
|
|
2518
|
+
|
|
2519
|
+
|
|
2520
|
+
def ydh_to_date(year, day, hour):
|
|
2521
|
+
dates = []
|
|
2522
|
+
for y, d, h in zip(year, day, hour):
|
|
2523
|
+
ts = pd.Timestamp(y, 1, 1) + pd.Timedelta(days=d-1, hours=h)
|
|
2524
|
+
dates.append(ts)
|
|
2525
|
+
return pd.DatetimeIndex(dates)
|
|
2526
|
+
|
|
2527
|
+
|
|
2528
|
+
def _split_outside_group(s, splitchar, open='(', close=')'):
|
|
2529
|
+
out = []
|
|
2530
|
+
grp_cnt = 0
|
|
2531
|
+
start = 0
|
|
2532
|
+
for i, c in enumerate(s):
|
|
2533
|
+
if c == splitchar and grp_cnt == 0:
|
|
2534
|
+
out.append(s[start:i+1])
|
|
2535
|
+
start = i+1
|
|
2536
|
+
elif c == open:
|
|
2537
|
+
grp_cnt += 1
|
|
2538
|
+
elif c == close:
|
|
2539
|
+
grp_cnt -= 1
|
|
2540
|
+
|
|
2541
|
+
if grp_cnt < 0:
|
|
2542
|
+
raise ValueError('Given string has an unmatched {} at position {}'.format(close, i))
|
|
2543
|
+
|
|
2544
|
+
# Group count should be 0. If not, then there were more open characters than closing characters
|
|
2545
|
+
if grp_cnt > 0:
|
|
2546
|
+
raise ValueError('Given string had an unmatched {}'.format(open))
|
|
2547
|
+
|
|
2548
|
+
# Handle the last piece. Check that we haven't just added something, i.e. that the split character isn't the last
|
|
2549
|
+
# one in the string. If it is, then we should add an empty element
|
|
2550
|
+
if start == len(s) + 1:
|
|
2551
|
+
out.append('')
|
|
2552
|
+
else:
|
|
2553
|
+
out.append(s[start:])
|
|
2554
|
+
|
|
2555
|
+
return out
|
|
2556
|
+
|
|
2557
|
+
|
|
2558
|
+
def fortran_fmt_to_fwf_tuples(fmt_str):
|
|
2559
|
+
fmt_str = fmt_str.strip()
|
|
2560
|
+
if not fmt_str.startswith('(') or not fmt_str.endswith(')'):
|
|
2561
|
+
raise ValueError('The first and last non-whitespace characters of the format string must be open and close '
|
|
2562
|
+
'parentheses, respectively.')
|
|
2563
|
+
fmt_parts = _split_outside_group(fmt_str[1:-1], ',')
|
|
2564
|
+
colspecs = []
|
|
2565
|
+
types = []
|
|
2566
|
+
|
|
2567
|
+
idx = 0
|
|
2568
|
+
for part in fmt_parts:
|
|
2569
|
+
if '(' in part:
|
|
2570
|
+
match = re.match(r'(\d*)(\(.+\))', part)
|
|
2571
|
+
repeats = 1 if len(match.group(1)) == 0 else int(match.group(1))
|
|
2572
|
+
subfmt = match.group(2)
|
|
2573
|
+
subspecs, subtypes = fortran_fmt_to_fwf_tuples(subfmt)
|
|
2574
|
+
|
|
2575
|
+
for i in range(repeats):
|
|
2576
|
+
for start, stop in subspecs:
|
|
2577
|
+
colspecs.append((idx+start, idx+stop))
|
|
2578
|
+
types += subtypes
|
|
2579
|
+
idx += stop
|
|
2580
|
+
elif re.match(r'\d+x', part):
|
|
2581
|
+
# In Fortran, specifiers like "1x" mean that n spaces do not contain information and are just spacers
|
|
2582
|
+
nspaces = int(re.match(r'\d+', part).group(), re.IGNORECASE)
|
|
2583
|
+
idx += nspaces
|
|
2584
|
+
else:
|
|
2585
|
+
match = re.match(r'(\d*)([a-z])(\d+)', part, re.IGNORECASE)
|
|
2586
|
+
repeats = 1 if len(match.group(1)) == 0 else int(match.group(1))
|
|
2587
|
+
width = int(match.group(3))
|
|
2588
|
+
|
|
2589
|
+
for i in range(repeats):
|
|
2590
|
+
colspecs.append((idx, idx+width))
|
|
2591
|
+
types.append(match.group(2))
|
|
2592
|
+
idx += width
|
|
2593
|
+
|
|
2594
|
+
return colspecs, types
|
|
2595
|
+
|
|
2596
|
+
|
|
2597
|
+
def make_geos_date_ranges(dates):
|
|
2598
|
+
geos_tdel = pd.Timedelta(hours=3)
|
|
2599
|
+
geos_dates = dates.round('3H').unique().sort_values()
|
|
2600
|
+
date_ranges = []
|
|
2601
|
+
|
|
2602
|
+
range_start = geos_dates[0]
|
|
2603
|
+
last_date = geos_dates[0]
|
|
2604
|
+
|
|
2605
|
+
for d in geos_dates[1:]:
|
|
2606
|
+
if d - last_date > geos_tdel:
|
|
2607
|
+
# If the next time is more than the GEOS time step in the future, then the current date range is done.
|
|
2608
|
+
# Append it, remembering that the end date is exclusive
|
|
2609
|
+
date_ranges.append((range_start, last_date + geos_tdel))
|
|
2610
|
+
range_start = d
|
|
2611
|
+
last_date = d
|
|
2612
|
+
|
|
2613
|
+
# There will always be a last date range, even if we added a date range in the last loop (in that case only the last
|
|
2614
|
+
# time will go in the final date range)
|
|
2615
|
+
date_ranges.append((range_start, last_date+geos_tdel))
|
|
2616
|
+
return date_ranges
|
|
2617
|
+
|
|
2618
|
+
|
|
2619
|
+
def get_runlog_geos_date_ranges(rldf):
|
|
2620
|
+
rl_dates = ydh_to_date(rldf['Year'], rldf['Day'], rldf['Hour'])
|
|
2621
|
+
return make_geos_date_ranges(rl_dates)
|
|
2622
|
+
|
|
2623
|
+
|
|
2624
|
+
def compute_file_checksum(file_path):
|
|
2625
|
+
"""Compute an MD5 checksum on the given file
|
|
2626
|
+
|
|
2627
|
+
Parameters
|
|
2628
|
+
----------
|
|
2629
|
+
file_path : str or Path
|
|
2630
|
+
Path to the file
|
|
2631
|
+
|
|
2632
|
+
Returns
|
|
2633
|
+
-------
|
|
2634
|
+
str
|
|
2635
|
+
The hexadecimal representation of the checksum
|
|
2636
|
+
"""
|
|
2637
|
+
checksum = md5()
|
|
2638
|
+
with open(file_path, 'rb') as f:
|
|
2639
|
+
while True:
|
|
2640
|
+
block = f.read(1000000)
|
|
2641
|
+
if len(block) == 0:
|
|
2642
|
+
break
|
|
2643
|
+
checksum.update(block)
|
|
2644
|
+
return checksum.hexdigest()
|
|
2645
|
+
|
|
2646
|
+
|
|
2647
|
+
def compute_bytes_checksum(bytes):
|
|
2648
|
+
"""Compute an MD5 checksum on the given bytes
|
|
2649
|
+
|
|
2650
|
+
Parameters
|
|
2651
|
+
----------
|
|
2652
|
+
bytes
|
|
2653
|
+
A byte string. To compute the checksum of a string,
|
|
2654
|
+
use `.encode()` to convert it to bytes.
|
|
2655
|
+
|
|
2656
|
+
Returns
|
|
2657
|
+
-------
|
|
2658
|
+
str
|
|
2659
|
+
The hexadecimal representation of the checksum
|
|
2660
|
+
"""
|
|
2661
|
+
checksum = md5()
|
|
2662
|
+
checksum.update(bytes)
|
|
2663
|
+
return checksum.hexdigest()
|