ginput 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ginput/__init__.py +4 -0
- ginput/common_utils/__init__.py +0 -0
- ginput/common_utils/ggg_logging.py +62 -0
- ginput/common_utils/ioutils.py +304 -0
- ginput/common_utils/mod_constants.py +24 -0
- ginput/common_utils/mod_utils.py +2663 -0
- ginput/common_utils/readers.py +477 -0
- ginput/common_utils/run_utils.py +57 -0
- ginput/common_utils/sat_utils.py +67 -0
- ginput/common_utils/versioning.py +251 -0
- ginput/common_utils/writers.py +377 -0
- ginput/data/.gitignore +1 -0
- ginput/data/GEOS_FPIT_lat_vs_theta_2018_500-700hPa.nc +0 -0
- ginput/data/ML_monthly_obs_ch4.txt +498 -0
- ginput/data/ML_monthly_obs_co.txt +412 -0
- ginput/data/ML_monthly_obs_co2.txt +597 -0
- ginput/data/ML_monthly_obs_n2o.txt +330 -0
- ginput/data/SMO_monthly_obs_ch4.txt +499 -0
- ginput/data/SMO_monthly_obs_co.txt +416 -0
- ginput/data/SMO_monthly_obs_co2.txt +613 -0
- ginput/data/SMO_monthly_obs_n2o.txt +330 -0
- ginput/data/ace_fn2o_lut.nc +0 -0
- ginput/data/age_spectra/age.midlat.txt +47 -0
- ginput/data/age_spectra/age.tropics.txt +46 -0
- ginput/data/age_spectra/age.vortex.txt +31 -0
- ginput/data/age_spectra/agespec.midlat.txt +47 -0
- ginput/data/age_spectra/agespec.tropics.txt +46 -0
- ginput/data/age_spectra/agespec.vortex.txt +31 -0
- ginput/data/age_spectra/time.txt +3001 -0
- ginput/data/ch4_hf_slopes.nc +0 -0
- ginput/data/ch4_strat_lut.nc +0 -0
- ginput/data/clams_age_clim_scaled.nc +0 -0
- ginput/data/co2_strat_lut.nc +0 -0
- ginput/data/fo2_data/.gitignore +2 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241118T180747/monthly_o2_ljo.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/co2_annmean_gl.txt +84 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_alt.csv +455 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_cgo.csv +431 -0
- ginput/data/fo2_data/fo2_inputs_20241219T112756/monthly_o2_ljo.csv +455 -0
- ginput/data/hf_strat_lut.nc +0 -0
- ginput/data/isotopologs.dat +199 -0
- ginput/data/meso_co_lut.nc +0 -0
- ginput/data/n2o_ch4_acefts.nc +0 -0
- ginput/data/n2o_strat_lut.nc +0 -0
- ginput/data/o2_mean_dmf.dat +52 -0
- ginput/data/summer_35N.vmr +182 -0
- ginput/data/washenfelder03_table_s3.txt +83 -0
- ginput/download/__init__.py +0 -0
- ginput/download/download_utils.py +46 -0
- ginput/download/get_GEOS5.py +367 -0
- ginput/download/get_MERRA2.py +177 -0
- ginput/download/get_NOAA_flask_data.py +103 -0
- ginput/download/get_fo2_data.py +140 -0
- ginput/mod_maker/__init__.py +0 -0
- ginput/mod_maker/mod_maker.py +2470 -0
- ginput/mod_maker/slantify.py +450 -0
- ginput/mod_maker/tccon_sites.py +570 -0
- ginput/priors/__init__.py +0 -0
- ginput/priors/acos_interface.py +1376 -0
- ginput/priors/automation.py +313 -0
- ginput/priors/backend_analysis/__init__.py +0 -0
- ginput/priors/backend_analysis/ace_fts_analysis.py +1418 -0
- ginput/priors/backend_analysis/ace_fts_val.py +452 -0
- ginput/priors/backend_analysis/acos_comparison.py +550 -0
- ginput/priors/backend_analysis/aircraft_aircore_val.py +1219 -0
- ginput/priors/backend_analysis/aircraft_preprocessing.py +295 -0
- ginput/priors/backend_analysis/atmos_mkiv_val.py +116 -0
- ginput/priors/backend_analysis/backend_driver.py +90 -0
- ginput/priors/backend_analysis/backend_utils.py +358 -0
- ginput/priors/backend_analysis/clams.py +71 -0
- ginput/priors/backend_analysis/create_test_priors.py +853 -0
- ginput/priors/backend_analysis/geos_theta_lat.py +594 -0
- ginput/priors/backend_analysis/mls_val.py +217 -0
- ginput/priors/backend_analysis/retrieval_obs_val.py +1306 -0
- ginput/priors/fo2_prep.py +311 -0
- ginput/priors/map_maker.py +193 -0
- ginput/priors/mlo_smo_prep.py +1146 -0
- ginput/priors/tccon_priors.py +3482 -0
- ginput-1.3.0.dist-info/LICENSE.txt +203 -0
- ginput-1.3.0.dist-info/METADATA +125 -0
- ginput-1.3.0.dist-info/RECORD +86 -0
- ginput-1.3.0.dist-info/WHEEL +5 -0
- ginput-1.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from enum import Enum
|
|
4
|
+
import netCDF4 as ncdf
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import re
|
|
8
|
+
import shutil
|
|
9
|
+
|
|
10
|
+
from .mod_utils import compute_file_checksum
|
|
11
|
+
from .ioutils import make_dependent_file_hash
|
|
12
|
+
|
|
13
|
+
from typing import Union, Dict, Optional, Sequence
|
|
14
|
+
|
|
15
|
+
class GeosSource(Enum):
|
|
16
|
+
FPIT = 'fpit'
|
|
17
|
+
FP = 'fp'
|
|
18
|
+
IT = 'it'
|
|
19
|
+
UNKNOWN = 'UNKNOWN'
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def from_str(cls, s):
|
|
23
|
+
if s in {'fpit', 'fp', 'it'}:
|
|
24
|
+
return cls(s)
|
|
25
|
+
else:
|
|
26
|
+
return cls.UNKNOWN
|
|
27
|
+
|
|
28
|
+
class GeosVersion:
|
|
29
|
+
def __init__(self, version_str: str, source: Union[str, GeosSource], file_name: str, md5_checksum: str):
|
|
30
|
+
self.version_str = version_str
|
|
31
|
+
self.file_name = file_name
|
|
32
|
+
self.md5_checksum = md5_checksum
|
|
33
|
+
if isinstance(source, str):
|
|
34
|
+
self.source = GeosSource.from_str(source)
|
|
35
|
+
else:
|
|
36
|
+
self.source = source
|
|
37
|
+
|
|
38
|
+
def __eq__(self, other):
|
|
39
|
+
if not isinstance(other, self.__class__):
|
|
40
|
+
return False
|
|
41
|
+
else:
|
|
42
|
+
return self.version_str == other.version_str and self.source == other.source
|
|
43
|
+
|
|
44
|
+
def __repr__(self):
|
|
45
|
+
clsname = self.__class__.__name__
|
|
46
|
+
return f'<{clsname}({self.source}, {self.version_str})>'
|
|
47
|
+
|
|
48
|
+
def __str__(self):
|
|
49
|
+
return f'{self.source.value} (GEOS v{self.version_str}) : {self.file_name} : {self.md5_checksum}'
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def from_str(cls, s):
|
|
53
|
+
parts = [x.strip() for x in s.split(':')]
|
|
54
|
+
if len(parts) != 3:
|
|
55
|
+
raise ValueError(f'"{s}" does not contain three substrings separated by colons')
|
|
56
|
+
|
|
57
|
+
version, file_name, checksum = parts
|
|
58
|
+
m = re.match(r'(\w+) \(GEOS v(\d+\.\d+\.\d+)\)', version)
|
|
59
|
+
if m is None:
|
|
60
|
+
raise ValueError(f'"{s}" does not match the expected format of a ginput GEOS version string')
|
|
61
|
+
else:
|
|
62
|
+
vstr = m.group(2)
|
|
63
|
+
src_str = m.group(1)
|
|
64
|
+
return cls(vstr, src_str, file_name, checksum)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def from_nc_file(cls, nc_file):
|
|
68
|
+
file_name = os.path.basename(nc_file)
|
|
69
|
+
file_hash = compute_file_checksum(nc_file)
|
|
70
|
+
with ncdf.Dataset(nc_file) as nc_dset:
|
|
71
|
+
vstr = nc_dset.VersionID
|
|
72
|
+
granule = nc_dset.GranuleID
|
|
73
|
+
# Assume a granule name like "GEOS.it.asm.asm_inst_3hr_glo_L576x361_v72.GEOS5294.2008-01-01T0000.V01.nc4"
|
|
74
|
+
# where the "source" is the second component as divided by "."s.
|
|
75
|
+
source_str = granule.split('.')[1]
|
|
76
|
+
return cls(vstr, source_str, file_name, file_hash)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def update_versioned_file_header(prev_file: Union[str, Path, Sequence[str]], new_data_descr: str, program_descr: str, source_files: Dict[str, Union[str, Path]],
|
|
80
|
+
header_comment_symbol: str = '#', insert_line_index: int = 0):
|
|
81
|
+
"""For a file which can have data appended/updated, include a new entry in the HISTORY part of the header.
|
|
82
|
+
|
|
83
|
+
This will return a list of header lines. If ``prev_file`` is ``None``, then the returned list will have
|
|
84
|
+
a HISTORY block with an entry for the initial creation of the file. Otherwise, the returned list will have
|
|
85
|
+
the header lines from ``prev_file`` with additional lines for the new change added. If there was already
|
|
86
|
+
a HISTORY section in the header, the new lines are added to the end of that section. Otherwise, they will
|
|
87
|
+
be inserted at the ``insert_line_index``, i.e. ``header[insert_line_index]`` will be the first new line
|
|
88
|
+
added, and any lines from ``prev_header[insert_line_index:]`` will come after the new lines. In the latter
|
|
89
|
+
case, the "HISTORY" and "END HISTORY" lines that denote a history section will be added as well.
|
|
90
|
+
|
|
91
|
+
Parameters
|
|
92
|
+
----------
|
|
93
|
+
prev_file
|
|
94
|
+
Path to the previous version of the file or, if this is the first time this file is written,
|
|
95
|
+
the list of header lines to be written to the new file (excluding the history block).
|
|
96
|
+
|
|
97
|
+
new_data_descr
|
|
98
|
+
A brief description of what data was added, this will be included in the first line of the new history
|
|
99
|
+
entry.
|
|
100
|
+
|
|
101
|
+
program_descr
|
|
102
|
+
A brief description of what program added the new data, this will be included in the first line of the new
|
|
103
|
+
history entry.
|
|
104
|
+
|
|
105
|
+
source_files
|
|
106
|
+
A dictionary where the keys are short descriptions of the source files used to compute the new data and
|
|
107
|
+
the values are the paths to those files. Each source file will be added as a line in the new history
|
|
108
|
+
section with its SHA1 hash.
|
|
109
|
+
|
|
110
|
+
header_comment_symbol
|
|
111
|
+
What symbol indicates that a line is a comment; this will be added to the beginning of new lines created
|
|
112
|
+
for the header and, when reading a previous file, lines that begin with this string will be included as
|
|
113
|
+
previous header lines.
|
|
114
|
+
|
|
115
|
+
insert_line_index
|
|
116
|
+
The index in the list of header lines to insert the history block if ``prev_file`` is given, but it has
|
|
117
|
+
no history block in the header. This input is ignored if ``prev_file`` is ``None`` or there is a history
|
|
118
|
+
block in its header.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
header_lines
|
|
123
|
+
A list of header lines that combines the previous header (read from the previous file or provided as
|
|
124
|
+
the ``prev_file`` argument) and the new history section.
|
|
125
|
+
"""
|
|
126
|
+
# Create the new history lines, including source files and (if given) the previous file
|
|
127
|
+
cs = header_comment_symbol
|
|
128
|
+
now = datetime.now(timezone.utc).astimezone()
|
|
129
|
+
|
|
130
|
+
new_history = [
|
|
131
|
+
f'{cs} Added {new_data_descr} using {program_descr} at {now:%Y-%m-%d %H:%M %Z}. Source files:\n'
|
|
132
|
+
]
|
|
133
|
+
if isinstance(prev_file, (str, Path)):
|
|
134
|
+
new_history.append(f'{cs} - Previous file: {prev_file} (SHA1 = {make_dependent_file_hash(prev_file)})\n')
|
|
135
|
+
for file_type, file_path in source_files.items():
|
|
136
|
+
new_history.append(
|
|
137
|
+
f'{cs} - {file_type}: {file_path} (SHA1 = {make_dependent_file_hash(file_path)})\n'
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# If given a path to a previous file, read in its header
|
|
141
|
+
if isinstance(prev_file, (str, Path)):
|
|
142
|
+
with open(prev_file) as f:
|
|
143
|
+
header = []
|
|
144
|
+
for idx, line in enumerate(f):
|
|
145
|
+
if line.startswith(header_comment_symbol):
|
|
146
|
+
header.append(line)
|
|
147
|
+
else:
|
|
148
|
+
break
|
|
149
|
+
else:
|
|
150
|
+
header = prev_file
|
|
151
|
+
|
|
152
|
+
# Find the end of the history block, if present.
|
|
153
|
+
end_hist_index = None
|
|
154
|
+
for idx, line in enumerate(header):
|
|
155
|
+
if line.strip() == f'{cs} END HISTORY':
|
|
156
|
+
end_hist_index = idx
|
|
157
|
+
|
|
158
|
+
if end_hist_index is not None:
|
|
159
|
+
# File already has a history block, so we want to add to its end
|
|
160
|
+
insert_line_index = end_hist_index
|
|
161
|
+
else:
|
|
162
|
+
# File does not have a history block, so we need to include the block start/end indicator lines
|
|
163
|
+
new_history.insert(0, f'{cs} HISTORY:\n')
|
|
164
|
+
new_history.append(f'{cs} END HISTORY\n')
|
|
165
|
+
|
|
166
|
+
return header[:insert_line_index] + new_history + header[insert_line_index:]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class RollingBackup(ABC):
|
|
170
|
+
"""Base class for various scheme of making a series of backups.
|
|
171
|
+
"""
|
|
172
|
+
def make_rolling_backup(self, src_file: Union[str, Path], max_num_backups: Optional[int] = None) -> Path:
|
|
173
|
+
"""Backup a file, creating a sequence of backups with an optional maximum number of backup files.
|
|
174
|
+
|
|
175
|
+
Parameters
|
|
176
|
+
----------
|
|
177
|
+
src_file
|
|
178
|
+
The file to back up
|
|
179
|
+
|
|
180
|
+
max_num_backups
|
|
181
|
+
The maximum number of backup files to keep (not counting the original). If not given, there
|
|
182
|
+
will be no limit on the number of backup files.
|
|
183
|
+
|
|
184
|
+
Returns
|
|
185
|
+
-------
|
|
186
|
+
dest_file
|
|
187
|
+
The path to the backup created.
|
|
188
|
+
"""
|
|
189
|
+
src_file = Path(src_file)
|
|
190
|
+
prev_backups = self._existing_backups(src_file)
|
|
191
|
+
self._shuffle_backups(prev_backups, max_num_backups)
|
|
192
|
+
dest_file = self._name(src_file)
|
|
193
|
+
shutil.copy2(str(src_file), str(dest_file))
|
|
194
|
+
return dest_file
|
|
195
|
+
|
|
196
|
+
def _shuffle_backups(self, prev_backups: Sequence[Path], max_num_backups: Optional[int]):
|
|
197
|
+
"""Rename and/or delete previous backups to make room in the sequence for a new file.
|
|
198
|
+
|
|
199
|
+
The default implementation just removes enough previous backups
|
|
200
|
+
|
|
201
|
+
Parameters
|
|
202
|
+
----------
|
|
203
|
+
prev_backups
|
|
204
|
+
The list of previous backup files, ordered from oldest to newest.
|
|
205
|
+
|
|
206
|
+
max_num_backups
|
|
207
|
+
The maximum number of backup files to keep (not counting the original). If not given, there
|
|
208
|
+
will be no limit on the number of backup files.
|
|
209
|
+
"""
|
|
210
|
+
if max_num_backups is None or len(prev_backups) < (max_num_backups - 1):
|
|
211
|
+
return
|
|
212
|
+
|
|
213
|
+
n_to_remove = len(prev_backups) - max_num_backups + 1
|
|
214
|
+
for prev_backup in prev_backups[:n_to_remove]:
|
|
215
|
+
os.remove(prev_backup)
|
|
216
|
+
|
|
217
|
+
@abstractmethod
|
|
218
|
+
def _name(self, src_file: Path) -> Path:
|
|
219
|
+
"""Return the path to copy the new backup to.
|
|
220
|
+
"""
|
|
221
|
+
pass
|
|
222
|
+
|
|
223
|
+
@abstractmethod
|
|
224
|
+
def _existing_backups(self, src_file: Path) -> Sequence[Path]:
|
|
225
|
+
"""Return a list of existing backup files, ordered oldest to newest.
|
|
226
|
+
"""
|
|
227
|
+
pass
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
class RollingBackupByDate(RollingBackup):
|
|
231
|
+
"""A rolling backup implementation that appends ".bak{DATE}" to backup file name
|
|
232
|
+
|
|
233
|
+
Parameters
|
|
234
|
+
----------
|
|
235
|
+
date_fmt
|
|
236
|
+
A format string for :func:`datetime.strftime` that will be used to format the
|
|
237
|
+
current time when naming the backup file.
|
|
238
|
+
"""
|
|
239
|
+
def __init__(self, date_fmt: str = '%Y%m%dT%H%M%S'):
|
|
240
|
+
self._date_fmt = date_fmt
|
|
241
|
+
|
|
242
|
+
def _name(self, src_file: Path) -> Path:
|
|
243
|
+
curr_name = src_file.name
|
|
244
|
+
date_str = datetime.now().strftime(self._date_fmt)
|
|
245
|
+
new_name = f'{curr_name}.bak{date_str}'
|
|
246
|
+
return src_file.with_name(new_name)
|
|
247
|
+
|
|
248
|
+
def _existing_backups(self, src_file: Path) -> Sequence[Path]:
|
|
249
|
+
src_dir = src_file.parent
|
|
250
|
+
pattern = f'{src_file.name}.bak*'
|
|
251
|
+
return sorted(src_dir.glob(pattern))
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import netCDF4 as ncdf
|
|
2
|
+
import numpy as np
|
|
3
|
+
import os
|
|
4
|
+
from warnings import warn
|
|
5
|
+
|
|
6
|
+
# Allow the ginput.common_utils.writers module to be imported if there's an issue interacting with the C library. This
|
|
7
|
+
# can happen if it's not installed on the system, or when called from a Jupyter notebook.
|
|
8
|
+
try:
|
|
9
|
+
from cfunits import Units
|
|
10
|
+
except (AssertionError, AttributeError, FileNotFoundError) as e:
|
|
11
|
+
warn(f'Could not import cfunits due to a(n) {type(e).__name__}. Will not be able to enforce CF units conventions.')
|
|
12
|
+
cfunits_imported = False
|
|
13
|
+
else:
|
|
14
|
+
cfunits_imported = True
|
|
15
|
+
|
|
16
|
+
from .ggg_logging import logger
|
|
17
|
+
from . import mod_utils, mod_constants, ioutils, readers
|
|
18
|
+
from ..mod_maker import tccon_sites
|
|
19
|
+
from .. import __version__
|
|
20
|
+
|
|
21
|
+
_map_scale_factors = {'co2': 1e6, 'n2o': 1e9, 'co': 1e9, 'ch4': 1e9, 'hf': 1e12}
|
|
22
|
+
_map_text_units = {'Height': 'km', 'Temp': 'K', 'Pressure': 'hPa', 'Density': 'molecules_cm3', 'h2o': 'parts',
|
|
23
|
+
'hdo': 'parts', 'co2': 'ppm', 'n2o': 'ppb', 'co': 'ppb', 'ch4': 'ppb', 'hf': 'ppt', 'o2': 'parts',
|
|
24
|
+
'gravity': 'm_s2'}
|
|
25
|
+
# For units whose old text names are not recognizable under the CF convention, replace them with ones that are
|
|
26
|
+
_map_canonical_units = _map_text_units.copy()
|
|
27
|
+
_map_canonical_units.update({'Density': 'molecules/cm^3',
|
|
28
|
+
'h2o': 'mol/mol',
|
|
29
|
+
'hdo': 'mol/mol',
|
|
30
|
+
'o2': 'mol/mol',
|
|
31
|
+
'gravity': 'm/s^2'})
|
|
32
|
+
_map_standard_names = {'Height': 'altitude',
|
|
33
|
+
'Temp': 'air_temperature',
|
|
34
|
+
'Pressure': 'air_pressure',
|
|
35
|
+
'Density': 'air_number_density',
|
|
36
|
+
'h2o': 'water_{w_or_d}_mole_fraction',
|
|
37
|
+
'hdo': 'heavy_water_{w_or_d}_mole_fraction',
|
|
38
|
+
'co2': 'carbon_dioxide_{w_or_d}_mole_fraction',
|
|
39
|
+
'n2o': 'nitrous_oxide_{w_or_d}_mole_fraction',
|
|
40
|
+
'co': 'carbon_monoxide_{w_or_d}_mole_fraction',
|
|
41
|
+
'ch4': 'methane_{w_or_d}_mole_fraction',
|
|
42
|
+
'hf': 'hydrofluoric_acid_{w_or_d}_mole_fraction',
|
|
43
|
+
'o2': 'oxygen_{w_or_d}_mole_fraction',
|
|
44
|
+
'gravity': 'gravitational_acceleration'}
|
|
45
|
+
|
|
46
|
+
_map_var_mapping = {'Temperature': 'Temp'}
|
|
47
|
+
_map_var_order = ('Height', 'Temp', 'Pressure', 'Density', 'h2o', 'hdo', 'co2', 'n2o', 'co', 'ch4', 'hf', 'o2', 'gravity')
|
|
48
|
+
|
|
49
|
+
_float_fmt = '{:8.3f}'
|
|
50
|
+
_exp_fmt = '{:10.3E}'
|
|
51
|
+
_map_var_formats = {'Height': _float_fmt, 'Temp': _float_fmt, 'Pressure': _exp_fmt, 'Density': _exp_fmt,
|
|
52
|
+
'h2o': _exp_fmt, 'hdo': _exp_fmt, 'co2': _float_fmt, 'n2o': _float_fmt, 'co': _exp_fmt,
|
|
53
|
+
'ch4': '{:7.1f}', 'hf': _float_fmt, 'o2': '{:7.4f}', 'gravity': '{:6.3f}'}
|
|
54
|
+
|
|
55
|
+
wmf_message = ['NOTE: The gas concentrations (including H2O) are WET MOLE FRACTIONS. If you require dry mole fractions,',
|
|
56
|
+
'you must calculate [H2O]_dry = (1/[H2O]_wet - 1)^-1 and then [gas]_dry = [gas]_wet * (1 + [H2O]_dry).']
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CFUnitsError(Exception):
|
|
60
|
+
def __init__(self, unit_string):
|
|
61
|
+
msg = '"{}" is not a CF-compliant unit'.format(unit_string)
|
|
62
|
+
super(CFUnitsError, self).__init__(msg)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _cfunits(unit_string, no_cfunits=False):
|
|
66
|
+
"""
|
|
67
|
+
Convert a units string to a CF-compliant one.
|
|
68
|
+
:param unit_string: the unit string to convert
|
|
69
|
+
:param no_cfunits: if True, then will not format unit strings if CFUnits failed to import. Has no effect if CFUnits
|
|
70
|
+
did import successfully.
|
|
71
|
+
:return: the converted unit string
|
|
72
|
+
:raises CFUnitsError: if the string cannot be made CF-compliant
|
|
73
|
+
"""
|
|
74
|
+
if not cfunits_imported:
|
|
75
|
+
if no_cfunits:
|
|
76
|
+
return unit_string
|
|
77
|
+
else:
|
|
78
|
+
raise ImportError('cfunits.Unit was not successfully imported. Use the no_cfunits keyword to ignore that '
|
|
79
|
+
'problem and skip making the units CF compliant.')
|
|
80
|
+
|
|
81
|
+
units = Units(unit_string).formatted()
|
|
82
|
+
if units is None:
|
|
83
|
+
raise CFUnitsError(unit_string)
|
|
84
|
+
else:
|
|
85
|
+
return units
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def write_map_from_vmr_mod(vmr_file, mod_file, map_output_dir, fmt='txt', wet_or_dry='wet', site_abbrev='xx',
|
|
89
|
+
no_cfunits=False):
|
|
90
|
+
"""
|
|
91
|
+
Create a .map file from a .vmr and .mod file
|
|
92
|
+
|
|
93
|
+
:param vmr_file: the path to the .vmr file to read the gas concentrations from
|
|
94
|
+
:param mod_file: the path to the .mod file to read the met variables from
|
|
95
|
+
:param map_output_dir: the directory to write the .map file to. It will automatically be given the correct name.
|
|
96
|
+
:param fmt: what format to write the .map files in, either "txt" for the original text files or "nc" for the new
|
|
97
|
+
netCDF files.
|
|
98
|
+
:param wet_or_dry: whether to write wet or dry mole fractions.
|
|
99
|
+
:param site_abbrev: the site abbreviation to go in the file name and netCDF attributes.
|
|
100
|
+
:param no_cfunits: if True, then will not format unit strings if CFUnits failed to import. Has no effect if CFUnits
|
|
101
|
+
did import successfully.
|
|
102
|
+
:return: none, writes .map or .map.nc file.
|
|
103
|
+
"""
|
|
104
|
+
if not os.path.isfile(vmr_file):
|
|
105
|
+
raise OSError('vmr_file "{}" does not exist'.format(vmr_file))
|
|
106
|
+
if not os.path.isdir(map_output_dir):
|
|
107
|
+
raise OSError('map_output_dir "{}" is not a directory'.format(map_output_dir))
|
|
108
|
+
if wet_or_dry not in ('wet', 'dry'):
|
|
109
|
+
raise ValueError('wet_or_dry must be "wet" or "dry"')
|
|
110
|
+
|
|
111
|
+
map_name = mod_utils.map_file_name_from_mod_vmr_files(site_abbrev, mod_file, vmr_file, fmt)
|
|
112
|
+
map_name = os.path.join(map_output_dir, map_name)
|
|
113
|
+
mapdat, obs_lat = _merge_and_convert_mod_vmr(vmr_file, mod_file, wet_or_dry=wet_or_dry)
|
|
114
|
+
|
|
115
|
+
if fmt in {'txt', 'text'}:
|
|
116
|
+
_write_text_map_file(mapdat=mapdat, obs_lat=obs_lat, map_file=map_name, wet_or_dry=wet_or_dry)
|
|
117
|
+
elif fmt in {'nc', 'netcdf'}:
|
|
118
|
+
moddat = readers.read_mod_file(mod_file)
|
|
119
|
+
_write_ncdf_map_file(mapdat=mapdat, obs_lat=obs_lat, obs_date=moddat['file']['datetime'], obs_site=site_abbrev,
|
|
120
|
+
file_lat=moddat['file']['lat'], file_lon=moddat['file']['lon'],
|
|
121
|
+
map_file=map_name, wet_or_dry=wet_or_dry, no_cfunits=no_cfunits)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _merge_and_convert_mod_vmr(vmr_file, mod_file, vmr_vars=('h2o', 'hdo', 'co2', 'n2o', 'co', 'ch4', 'hf', 'o2'),
|
|
125
|
+
mod_vars=('Height', 'Temperature', 'Pressure', 'Density', 'gravity'), wet_or_dry='wet'):
|
|
126
|
+
vmrdat = readers.read_vmr_file(vmr_file)
|
|
127
|
+
moddat = readers.read_mod_file(mod_file)
|
|
128
|
+
mapdat = dict()
|
|
129
|
+
|
|
130
|
+
# put the .mod variables (always on the GEOS native grid) on the same grid as the .vmr file (whatever that is).
|
|
131
|
+
obs_lat = moddat['constants']['obs_lat']
|
|
132
|
+
zgrid = vmrdat['profile']['altitude']
|
|
133
|
+
for mvar in mod_vars:
|
|
134
|
+
if mvar == 'gravity':
|
|
135
|
+
lat = np.broadcast_to([obs_lat], moddat['profile']['Height'].shape)
|
|
136
|
+
mapdat['gravity'], _ = mod_utils.gravity(lat, moddat['profile']['Height'])
|
|
137
|
+
elif mvar == 'Density':
|
|
138
|
+
mapdat['Density'] = mod_utils.number_density_air(moddat['profile']['Pressure'], moddat['profile']['Temperature'])
|
|
139
|
+
else:
|
|
140
|
+
mout = _map_var_mapping[mvar] if mvar in _map_var_mapping else mvar
|
|
141
|
+
scale = _map_scale_factors[mout] if mout in _map_scale_factors else 1
|
|
142
|
+
mapdat[mout] = moddat['profile'][mvar] * scale
|
|
143
|
+
|
|
144
|
+
mapdat = mod_utils.interp_to_zgrid(mapdat, zgrid=zgrid)
|
|
145
|
+
|
|
146
|
+
# the .vmr variables should be on the desired zgrid already since we took the zgrid from that file. However, we
|
|
147
|
+
# may need to change them from dry to wet VMRs. Setting the H2O dry mole fraction to 0 makes the mole fractions dry
|
|
148
|
+
# b/c there's no water, though it doesn't matter because of the `if wet_or_dry` in the loop.
|
|
149
|
+
h2o_dmf = vmrdat['profile']['h2o'] if wet_or_dry == 'wet' else 0
|
|
150
|
+
|
|
151
|
+
for vvar in vmr_vars:
|
|
152
|
+
scale = _map_scale_factors[vvar] if vvar in _map_scale_factors else 1
|
|
153
|
+
gas_dmf = vmrdat['profile'][vvar] * scale
|
|
154
|
+
if wet_or_dry == 'wet':
|
|
155
|
+
gas_dmf = mod_utils.dry2wet(gas_dmf, h2o_dmf)
|
|
156
|
+
mapdat[vvar] = gas_dmf
|
|
157
|
+
|
|
158
|
+
return mapdat, obs_lat
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _write_text_map_file(mapdat, obs_lat, map_file, wet_or_dry):
|
|
162
|
+
def iter_values_formats(irow):
|
|
163
|
+
for varname in _map_var_order:
|
|
164
|
+
yield mapdat[varname][irow], _map_var_formats[varname]
|
|
165
|
+
# First build up the header. A GGG2014 map file contains in the header:
|
|
166
|
+
# nheader/ncols
|
|
167
|
+
# its basename
|
|
168
|
+
# program versions
|
|
169
|
+
# reference to the TCCON wiki
|
|
170
|
+
# avogadro constant
|
|
171
|
+
# mass dry air
|
|
172
|
+
# mass H2O
|
|
173
|
+
# latitude
|
|
174
|
+
|
|
175
|
+
header = [os.path.basename(map_file),
|
|
176
|
+
'{:25} Version {:9} JLL, SR, MK'.format('GINPUT', __version__),
|
|
177
|
+
'Please see https://tccon-wiki.caltech.edu for a complete description of this file and its usage.']
|
|
178
|
+
|
|
179
|
+
# Usage and warnings
|
|
180
|
+
if wet_or_dry == 'wet':
|
|
181
|
+
header += wmf_message
|
|
182
|
+
|
|
183
|
+
# Constants
|
|
184
|
+
header.append('Avogadro (molecules/mole): {}'.format(mod_constants.avogadro))
|
|
185
|
+
header.append('Mass_Dry_Air (kg/mole): {}'.format(mod_constants.mass_dry_air))
|
|
186
|
+
header.append('Mass_H2O (kg/mole): {}'.format(mod_constants.mass_h2o))
|
|
187
|
+
header.append('Latitude (degrees): {}'.format(obs_lat))
|
|
188
|
+
|
|
189
|
+
# Column headers
|
|
190
|
+
header.append(','.join(_map_var_order))
|
|
191
|
+
header.append(','.join(_map_text_units[v] for v in _map_var_order))
|
|
192
|
+
|
|
193
|
+
# Now prepend the number of header rows and data columns
|
|
194
|
+
header.insert(0, '{} {}'.format(len(header)+1, len(_map_var_order)))
|
|
195
|
+
|
|
196
|
+
# Begin writing
|
|
197
|
+
with open(map_file, 'w') as wobj:
|
|
198
|
+
for line in header:
|
|
199
|
+
wobj.write(line + '\n')
|
|
200
|
+
for i in range(mapdat['Height'].size):
|
|
201
|
+
line = ','.join(fmt.format(value) for value, fmt in iter_values_formats(i))
|
|
202
|
+
wobj.write(line + '\n')
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _write_ncdf_map_file(mapdat, obs_lat, obs_date, file_lat, file_lon, obs_site, map_file, wet_or_dry,
|
|
206
|
+
no_cfunits=False):
|
|
207
|
+
with ncdf.Dataset(map_file, 'w') as wobj:
|
|
208
|
+
alt_human_units = _map_canonical_units['Height']
|
|
209
|
+
alt_units = _cfunits(alt_human_units, no_cfunits=no_cfunits)
|
|
210
|
+
altdim = ioutils.make_ncdim_helper(wobj, 'altitude', mapdat['Height'],
|
|
211
|
+
units=alt_units, full_units=alt_human_units, long_name='altitude',
|
|
212
|
+
tccon_name='height')
|
|
213
|
+
|
|
214
|
+
# not actually used, but makes more sense to write these as variables/dimensions than attributes
|
|
215
|
+
ioutils.make_nctimedim_helper(wobj, 'time', np.array([obs_date]), time_units='hours', long_name='time')
|
|
216
|
+
ioutils.make_ncdim_helper(wobj, 'lat', np.array([obs_lat]), units='degrees_north', long_name='latitude')
|
|
217
|
+
|
|
218
|
+
for varname in _map_var_order:
|
|
219
|
+
if varname == 'Height':
|
|
220
|
+
# already defined as a coordinate
|
|
221
|
+
continue
|
|
222
|
+
|
|
223
|
+
human_units = _map_canonical_units[varname]
|
|
224
|
+
cf_units = _cfunits(human_units, no_cfunits=no_cfunits)
|
|
225
|
+
std_name = _map_standard_names[varname].format(w_or_d=wet_or_dry)
|
|
226
|
+
ioutils.make_ncvar_helper(wobj, varname.lower(), mapdat[varname], dims=[altdim],
|
|
227
|
+
units=cf_units, full_units=human_units, long_name=std_name)
|
|
228
|
+
|
|
229
|
+
# finally the file attributes, including ginput version, constants used, WMF message, etc. Global CF attributes
|
|
230
|
+
# to include are "comment", "Conventions" (?), "history", "institution", "references", "source", and "title"
|
|
231
|
+
wobj.comment_full_units = 'The full_units attribute provides a human-readable counterpart to the ' \
|
|
232
|
+
'CF-compliant units attribute'
|
|
233
|
+
if wet_or_dry == 'wet':
|
|
234
|
+
wobj.comment_wet_mole_fractions = ' '.join(wmf_message)
|
|
235
|
+
wobj.comment_file_lat_lon = 'These are the latitude/longitude recorded in the input .mod file name. They ' \
|
|
236
|
+
'may be rounded to the nearest degree.'
|
|
237
|
+
wobj.contact = 'Joshua Laughner (jlaugh@caltech.edu)'
|
|
238
|
+
wobj.Conventions = 'CF-1.7'
|
|
239
|
+
wobj.institution = 'California Institute of Technology, Pasadena, CA, USA'
|
|
240
|
+
wobj.references = 'https://tccon-wiki.caltech.edu'
|
|
241
|
+
wobj.source = 'ginput version {}'.format(__version__)
|
|
242
|
+
wobj.title = 'GGG2020 TCCON prior profiles'
|
|
243
|
+
|
|
244
|
+
creation_note = 'ginput (commit {})'.format(mod_utils.vcs_commit_info()[0])
|
|
245
|
+
ioutils.add_creation_info(wobj, creation_note, creation_att_name='history')
|
|
246
|
+
|
|
247
|
+
# ggg-specific attributes
|
|
248
|
+
wobj.file_latitude = file_lat
|
|
249
|
+
wobj.file_longitude = file_lon
|
|
250
|
+
wobj.file_datetime = obs_date.strftime('%Y-%m-%d %H:%M:%S UTC')
|
|
251
|
+
wobj.tccon_site = obs_site
|
|
252
|
+
wobj.tccon_site_full_name = tccon_sites.site_dict[obs_site]['name'] if obs_site in tccon_sites.site_dict else 'N/A'
|
|
253
|
+
wobj.constant_avogadros_number = mod_constants.avogadro
|
|
254
|
+
wobj.constant_avogadros_number_units = 'molecules.mole-1' # CF convention would be '1.66053878316273e-24 1' which is just ugly
|
|
255
|
+
wobj.constant_mass_dry_air = mod_constants.mass_dry_air
|
|
256
|
+
wobj.constant_mass_dry_air_units = _cfunits('kg/mol', no_cfunits=no_cfunits)
|
|
257
|
+
wobj.constant_mass_h2o = mod_constants.mass_h2o
|
|
258
|
+
wobj.constant_mass_h2o_units = _cfunits('kg/mol', no_cfunits=no_cfunits)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def write_vmr_file(vmr_file, tropopause_alt, profile_date, profile_lat, profile_alt, profile_gases, gas_name_order=None,
|
|
262
|
+
extra_header_info=None):
|
|
263
|
+
"""
|
|
264
|
+
Write a new-style .vmr file (without seasonal cycle, secular trends, and latitudinal gradients
|
|
265
|
+
|
|
266
|
+
:param vmr_file: the path to write the .vmr file ar
|
|
267
|
+
:type vmr_file: str
|
|
268
|
+
|
|
269
|
+
:param tropopause_alt: the altitude of the tropopause, in kilometers
|
|
270
|
+
:type tropopause_alt: float
|
|
271
|
+
|
|
272
|
+
:param profile_date: the date of the profile
|
|
273
|
+
:type profile_date: datetime-like
|
|
274
|
+
|
|
275
|
+
:param profile_lat: the latitude of the profile (south is negative)
|
|
276
|
+
:type profile_lat: float
|
|
277
|
+
|
|
278
|
+
:param profile_alt: the altitude levels that the profiles are defined on, in kilometers
|
|
279
|
+
:type profile_alt: array-like
|
|
280
|
+
|
|
281
|
+
:param profile_gases: a dictionary of the prior profiles to write to the .vmr file.
|
|
282
|
+
:type profile_gases: dict(array)
|
|
283
|
+
|
|
284
|
+
:param gas_name_order: optional, a list/tuple specifying what order the gases are to be written in. If not given,
|
|
285
|
+
they will be written in whatever order the iteration through ``profile_gases`` defaults to. If given, then an
|
|
286
|
+
error is raised if any of the gas names listed here are not present in ``profile_gases`` (comparison is case-
|
|
287
|
+
insensitive). Any gases not listed here that are in ``profile_gases`` are skipped.
|
|
288
|
+
:type gas_name_order: list(str)
|
|
289
|
+
|
|
290
|
+
:param extra_header_info: optional, if given, must be a dictionary or list of lines to include at the end of the
|
|
291
|
+
header in the .vmr. If a list, must be a list of strings. If a dict, each line will be formatted as "key: value"
|
|
292
|
+
and the keys/values may be any type.
|
|
293
|
+
|
|
294
|
+
:return: none, writes the .vmr file.
|
|
295
|
+
"""
|
|
296
|
+
|
|
297
|
+
if np.ndim(profile_alt) != 1:
|
|
298
|
+
raise ValueError('profile_alt must be 1D')
|
|
299
|
+
|
|
300
|
+
if gas_name_order is None:
|
|
301
|
+
gas_name_order = [k for k in profile_gases.keys()]
|
|
302
|
+
|
|
303
|
+
if extra_header_info is None:
|
|
304
|
+
extra_header_info = []
|
|
305
|
+
elif isinstance(extra_header_info, dict):
|
|
306
|
+
extra_header_info = ['{}: {}'.format(k, v) for k, v in extra_header_info.items()]
|
|
307
|
+
|
|
308
|
+
gas_name_order_lower = [name.lower() for name in gas_name_order]
|
|
309
|
+
gas_name_mapping = {k: None for k in gas_name_order}
|
|
310
|
+
|
|
311
|
+
# Check that all the gases in the profile_gases dict are expected to be written.
|
|
312
|
+
for gas_name, gas_data in profile_gases.items():
|
|
313
|
+
if gas_name.lower() not in gas_name_order_lower:
|
|
314
|
+
logger.warning('Gas "{}" was not listed in the gas name order and will not be written to the .vmr '
|
|
315
|
+
'file'.format(gas_name))
|
|
316
|
+
elif np.shape(gas_data) != np.shape(profile_alt):
|
|
317
|
+
raise ValueError('Gas "{}" has a different shape ({}) than the altitude data ({})'.format(
|
|
318
|
+
gas_name, np.shape(gas_data), np.shape(profile_alt)
|
|
319
|
+
))
|
|
320
|
+
elif np.ndim(gas_data) != 1:
|
|
321
|
+
raise ValueError('Gas "{}" is not 1D'.format(gas_name))
|
|
322
|
+
else:
|
|
323
|
+
idx = gas_name_order_lower.index(gas_name.lower())
|
|
324
|
+
gas_name_mapping[gas_name_order[idx]] = gas_name
|
|
325
|
+
|
|
326
|
+
# Write the header, which starts with the number of header lines and data columns, then has the tropopause altitude,
|
|
327
|
+
# profile date as a decimal year, and profile latitude. I'm going to skip the secular trends, seasonal cycle, and
|
|
328
|
+
# latitude gradient because those are not necessary.
|
|
329
|
+
alt_fmt = '{:9.3f} '
|
|
330
|
+
gas_fmt = '{:.3E} '
|
|
331
|
+
table_header = ['Altitude'] + ['{:10}'.format(name) for name in gas_name_order]
|
|
332
|
+
header_lines = [' GINPUT_VERSION: {}'.format(__version__),
|
|
333
|
+
' ZTROP_VMR: {:.1f}'.format(tropopause_alt),
|
|
334
|
+
' DATE_VMR: {:.3f}'.format(mod_utils.date_to_decimal_year(profile_date)),
|
|
335
|
+
' LAT_VMR: {:.2f}'.format(profile_lat)] \
|
|
336
|
+
+ [' ' + l for l in extra_header_info] \
|
|
337
|
+
+ [' '.join(table_header)]
|
|
338
|
+
|
|
339
|
+
with open(vmr_file, 'w') as fobj:
|
|
340
|
+
_write_header(fobj, header_lines, len(gas_name_order) + 1)
|
|
341
|
+
for i in range(np.size(profile_alt)):
|
|
342
|
+
fobj.write(alt_fmt.format(profile_alt[i]))
|
|
343
|
+
for gas_name in gas_name_order:
|
|
344
|
+
if gas_name_mapping[gas_name] is not None:
|
|
345
|
+
gas_conc = profile_gases[gas_name_mapping[gas_name]][i]
|
|
346
|
+
else:
|
|
347
|
+
gas_conc = 0.0
|
|
348
|
+
fobj.write(gas_fmt.format(gas_conc))
|
|
349
|
+
fobj.write('\n')
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _write_header(fobj, header_lines, n_data_columns):
|
|
353
|
+
"""
|
|
354
|
+
Write a standard header to a GGG file.
|
|
355
|
+
|
|
356
|
+
A typical header in a GGG file begins with a line with two numbers: the number of header lines and the number of
|
|
357
|
+
data columns. This automatically formats that line, then writes the given lines.
|
|
358
|
+
|
|
359
|
+
:param fobj: File handle (i.e. object returned by :func:`open`)
|
|
360
|
+
:type fobj: :class:`_io.TextIO`
|
|
361
|
+
|
|
362
|
+
:param header_lines: a list of header lines (after the first) to write. Should be strings, each may or may not end
|
|
363
|
+
in newlines (both cases are handled).
|
|
364
|
+
:type header_lines: list(str)
|
|
365
|
+
|
|
366
|
+
:param n_data_columns: number of data columns in the main part of the file
|
|
367
|
+
:type n_data_columns: int
|
|
368
|
+
|
|
369
|
+
:return: none, writes to the file object.
|
|
370
|
+
"""
|
|
371
|
+
line1 = ' {} {}\n'.format(len(header_lines)+1, n_data_columns)
|
|
372
|
+
fobj.write(line1)
|
|
373
|
+
header_lines = [l if l.endswith('\n') else l + '\n' for l in header_lines]
|
|
374
|
+
fobj.writelines(header_lines)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
|
ginput/data/.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
o2_mean_dmf.dat
|
|
Binary file
|