vortex-nwp 2.0.0b1__py3-none-any.whl → 2.0.0b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +59 -45
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +940 -614
- vortex/algo/mpitools.py +802 -497
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +428 -225
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +544 -413
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +420 -271
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +210 -122
- vortex/nwp/algo/monitoring.py +132 -76
- vortex/nwp/algo/mpitools.py +321 -191
- vortex/nwp/algo/odbtools.py +617 -353
- vortex/nwp/algo/oopsroot.py +449 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +125 -59
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +311 -149
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +632 -298
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +402 -333
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1211 -631
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +377 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.0.0b2.dist-info/METADATA +66 -0
- vortex_nwp-2.0.0b2.dist-info/RECORD +142 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.0.0b2.dist-info}/top_level.txt +0 -0
vortex/nwp/tools/odb.py
CHANGED
|
@@ -8,7 +8,12 @@ from bronx.fancies import loggers
|
|
|
8
8
|
from bronx.stdtypes import date as bdate
|
|
9
9
|
import footprints
|
|
10
10
|
|
|
11
|
-
from vortex.algo.components import
|
|
11
|
+
from vortex.algo.components import (
|
|
12
|
+
AlgoComponentDecoMixin,
|
|
13
|
+
AlgoComponentError,
|
|
14
|
+
algo_component_deco_mixin_autodoc,
|
|
15
|
+
)
|
|
16
|
+
from vortex import config
|
|
12
17
|
from vortex.layout.dataflow import intent
|
|
13
18
|
|
|
14
19
|
from ..syntax.stdattrs import ArpIfsSimplifiedCycle
|
|
@@ -22,16 +27,18 @@ logger = loggers.getLogger(__name__)
|
|
|
22
27
|
class TimeSlots:
|
|
23
28
|
"""Handling of assimilation time slots."""
|
|
24
29
|
|
|
25
|
-
def __init__(
|
|
30
|
+
def __init__(
|
|
31
|
+
self, nslot=7, start="-PT3H", window="PT6H", chunk=None, center=True
|
|
32
|
+
):
|
|
26
33
|
if isinstance(nslot, str):
|
|
27
|
-
info = [x.strip() for x in nslot.split(
|
|
34
|
+
info = [x.strip() for x in nslot.split("/")]
|
|
28
35
|
nslot = info[0]
|
|
29
36
|
if len(info) > 1:
|
|
30
37
|
start = info[1]
|
|
31
38
|
if len(info) > 2:
|
|
32
39
|
window = info[2]
|
|
33
40
|
if len(info) > 3:
|
|
34
|
-
if re.match(
|
|
41
|
+
if re.match("^regular", info[3]):
|
|
35
42
|
center = False
|
|
36
43
|
else:
|
|
37
44
|
chunk = info[3]
|
|
@@ -41,7 +48,9 @@ class TimeSlots:
|
|
|
41
48
|
self.window = bdate.Period(window)
|
|
42
49
|
if chunk is None:
|
|
43
50
|
cslot = self.nslot - 1 if self.center else self.nslot
|
|
44
|
-
chunk =
|
|
51
|
+
chunk = (
|
|
52
|
+
"PT" + str((self.window.length // max(1, cslot)) // 60) + "M"
|
|
53
|
+
)
|
|
45
54
|
self.chunk = self.window if self.nslot < 2 else bdate.Period(chunk)
|
|
46
55
|
|
|
47
56
|
def __eq__(self, other):
|
|
@@ -50,31 +59,38 @@ class TimeSlots:
|
|
|
50
59
|
other = TimeSlots(other)
|
|
51
60
|
except ValueError:
|
|
52
61
|
pass
|
|
53
|
-
return (
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
62
|
+
return (
|
|
63
|
+
isinstance(other, TimeSlots)
|
|
64
|
+
and self.nslot == other.nslot
|
|
65
|
+
and self.center == other.center
|
|
66
|
+
and self.start == other.start
|
|
67
|
+
and self.window == other.window
|
|
68
|
+
and self.chunk == other.chunk
|
|
69
|
+
)
|
|
57
70
|
|
|
58
71
|
def __str__(self):
|
|
59
|
-
chunky = self.chunk.isoformat() if self.center else
|
|
60
|
-
return
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
chunky)
|
|
72
|
+
chunky = self.chunk.isoformat() if self.center else "regular"
|
|
73
|
+
return "{0.nslot:d}/{1:s}/{2:s}/{3:s}".format(
|
|
74
|
+
self, self.start.isoformat(), self.window.isoformat(), chunky
|
|
75
|
+
)
|
|
64
76
|
|
|
65
77
|
def __repr__(self, *args, **kwargs):
|
|
66
|
-
return super().__repr__()[:-1] +
|
|
78
|
+
return super().__repr__()[:-1] + " | {!s}>".format(self)
|
|
67
79
|
|
|
68
80
|
def as_slots(self):
|
|
69
81
|
"""Return a list of slots in seconds."""
|
|
70
82
|
if self.center:
|
|
71
|
-
slots = [
|
|
83
|
+
slots = [
|
|
84
|
+
self.chunk.length,
|
|
85
|
+
] * self.nslot
|
|
72
86
|
nb = self.window.length // self.chunk.length
|
|
73
87
|
if nb != self.nslot:
|
|
74
88
|
slots[0] = slots[-1] = self.chunk.length // 2
|
|
75
89
|
else:
|
|
76
90
|
islot = self.window.length // self.nslot
|
|
77
|
-
slots = [
|
|
91
|
+
slots = [
|
|
92
|
+
islot,
|
|
93
|
+
] * self.nslot
|
|
78
94
|
return slots
|
|
79
95
|
|
|
80
96
|
def as_centers_fromstart(self):
|
|
@@ -85,7 +101,9 @@ class TimeSlots:
|
|
|
85
101
|
for i in range(len(slots)):
|
|
86
102
|
fromstart.append(acc + slots[i] / 2)
|
|
87
103
|
acc += slots[i]
|
|
88
|
-
if self.center and (
|
|
104
|
+
if self.center and (
|
|
105
|
+
self.window.length // self.chunk.length != self.nslot
|
|
106
|
+
):
|
|
89
107
|
fromstart[0] = 0
|
|
90
108
|
fromstart[-1] = self.window.length
|
|
91
109
|
return [bdate.Period(seconds=t) for t in fromstart]
|
|
@@ -93,7 +111,9 @@ class TimeSlots:
|
|
|
93
111
|
def as_bounds(self, date):
|
|
94
112
|
"""Return time slots as a list of compact date values."""
|
|
95
113
|
date = bdate.Date(date)
|
|
96
|
-
boundlist = [
|
|
114
|
+
boundlist = [
|
|
115
|
+
date + self.start,
|
|
116
|
+
]
|
|
97
117
|
for x in self.as_slots():
|
|
98
118
|
boundlist.append(boundlist[-1] + x)
|
|
99
119
|
boundlist = [x.compact() for x in boundlist]
|
|
@@ -111,16 +131,21 @@ class TimeSlots:
|
|
|
111
131
|
|
|
112
132
|
def as_environment(self):
|
|
113
133
|
"""Return a dictionary of ready-to-export variables that describe the timeslots."""
|
|
114
|
-
thelen =
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
134
|
+
thelen = (
|
|
135
|
+
self.chunk.length // 60 if self.center and self.nslot > 1 else 0
|
|
136
|
+
)
|
|
137
|
+
return dict(
|
|
138
|
+
BATOR_WINDOW_LEN=self.window.length // 60,
|
|
139
|
+
BATOR_WINDOW_SHIFT=int(self.start.total_seconds()) // 60,
|
|
140
|
+
BATOR_SLOT_LEN=thelen,
|
|
141
|
+
BATOR_CENTER_LEN=thelen,
|
|
142
|
+
)
|
|
118
143
|
|
|
119
144
|
def as_file(self, date, filename):
|
|
120
145
|
"""Fill the specified ``filename`` wih the current list of time slots at this ``date``."""
|
|
121
|
-
with open(filename,
|
|
146
|
+
with open(filename, "w") as fd:
|
|
122
147
|
for x in self.as_bounds(date):
|
|
123
|
-
fd.write(str(x) +
|
|
148
|
+
fd.write(str(x) + "\n")
|
|
124
149
|
nbx = fd.tell()
|
|
125
150
|
return nbx
|
|
126
151
|
|
|
@@ -136,15 +161,23 @@ class OdbDriver:
|
|
|
136
161
|
self.cycle = cycle
|
|
137
162
|
self.sh = sh
|
|
138
163
|
if self.sh is None:
|
|
139
|
-
logger.critical(
|
|
164
|
+
logger.critical(
|
|
165
|
+
"%s created with a proper shell access [%s]",
|
|
166
|
+
self.__class__,
|
|
167
|
+
self,
|
|
168
|
+
)
|
|
140
169
|
self.env = env
|
|
141
170
|
if self.env is None:
|
|
142
|
-
logger.critical(
|
|
171
|
+
logger.critical(
|
|
172
|
+
"%s created with a proper environment access [%s]",
|
|
173
|
+
self.__class__,
|
|
174
|
+
self,
|
|
175
|
+
)
|
|
143
176
|
|
|
144
|
-
def setup(self, date, npool=1, nslot=1, iomethod=1, layout=
|
|
177
|
+
def setup(self, date, npool=1, nslot=1, iomethod=1, layout="ecma"):
|
|
145
178
|
"""Setup given environment with default ODB env variables."""
|
|
146
179
|
|
|
147
|
-
logger.info(
|
|
180
|
+
(logger.info("ODB: generic setup called."),)
|
|
148
181
|
self.env.update(
|
|
149
182
|
ODB_CMA=layout.upper(),
|
|
150
183
|
ODB_IO_METHOD=iomethod,
|
|
@@ -156,7 +189,7 @@ class OdbDriver:
|
|
|
156
189
|
ODB_REPRODUCIBLE_SEQNO=4,
|
|
157
190
|
ODB_STATIC_LINKING=1,
|
|
158
191
|
ODB_ANALYSIS_DATE=date.ymd,
|
|
159
|
-
ODB_ANALYSIS_TIME=date.hm +
|
|
192
|
+
ODB_ANALYSIS_TIME=date.hm + "00",
|
|
160
193
|
TO_ODB_ECMWF=0,
|
|
161
194
|
TO_ODB_SWAPOUT=0,
|
|
162
195
|
)
|
|
@@ -167,15 +200,17 @@ class OdbDriver:
|
|
|
167
200
|
ODB_IO_FILESIZE=128,
|
|
168
201
|
)
|
|
169
202
|
|
|
170
|
-
if self.sh.path.exists(
|
|
203
|
+
if self.sh.path.exists("IOASSIGN"):
|
|
171
204
|
self.env.default(
|
|
172
|
-
IOASSIGN=self.sh.path.abspath(
|
|
205
|
+
IOASSIGN=self.sh.path.abspath("IOASSIGN"),
|
|
173
206
|
)
|
|
174
207
|
|
|
175
208
|
def force_overwrite_method(self):
|
|
176
209
|
"""Force ODB_OVERWRITE_METHOD if necessary."""
|
|
177
|
-
if not int(self.env.get(
|
|
178
|
-
logger.info(
|
|
210
|
+
if not int(self.env.get("ODB_OVERWRITE_METHOD", 0)):
|
|
211
|
+
logger.info(
|
|
212
|
+
"ODB: Some input ODB databases are read-only. Setting ODB_OVERWRITE_METHOD to 1."
|
|
213
|
+
)
|
|
179
214
|
self.env.ODB_OVERWRITE_METHOD = 1
|
|
180
215
|
|
|
181
216
|
def _process_layout_dbpath(self, layout, dbpath=None):
|
|
@@ -191,19 +226,19 @@ class OdbDriver:
|
|
|
191
226
|
if env is None:
|
|
192
227
|
env = self.env
|
|
193
228
|
layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
|
|
194
|
-
logger.info(
|
|
195
|
-
env[
|
|
196
|
-
env[
|
|
229
|
+
logger.info("ODB: Fix %s path: %s", layout, dbpath)
|
|
230
|
+
env["ODB_SRCPATH_{:s}".format(layout)] = dbpath
|
|
231
|
+
env["ODB_DATAPATH_{:s}".format(layout)] = dbpath
|
|
197
232
|
|
|
198
233
|
@property
|
|
199
234
|
def _default_iocreate_path(self):
|
|
200
235
|
"""The location to the default create_ioassign utility."""
|
|
201
236
|
return self.sh.path.join(
|
|
202
|
-
from_config(section="nwp-tools", key="odb"),
|
|
203
|
-
get_from_config_w_default(
|
|
237
|
+
config.from_config(section="nwp-tools", key="odb"),
|
|
238
|
+
config.get_from_config_w_default(
|
|
204
239
|
section="nwp-tools",
|
|
205
240
|
key="iocreate_cmd",
|
|
206
|
-
default="create_ioassign"
|
|
241
|
+
default="create_ioassign",
|
|
207
242
|
),
|
|
208
243
|
)
|
|
209
244
|
|
|
@@ -211,16 +246,22 @@ class OdbDriver:
|
|
|
211
246
|
def _default_iomerge_path(self):
|
|
212
247
|
"""The location to the default merge_ioassign utility."""
|
|
213
248
|
return self.sh.path.join(
|
|
214
|
-
from_config(section="nwp-tools", key="odb"),
|
|
215
|
-
get_from_config_w_default(
|
|
249
|
+
config.from_config(section="nwp-tools", key="odb"),
|
|
250
|
+
config.get_from_config_w_default(
|
|
216
251
|
section="nwp-tools",
|
|
217
252
|
key="iomerge_cmd",
|
|
218
|
-
default="merge_ioassign"
|
|
253
|
+
default="merge_ioassign",
|
|
219
254
|
),
|
|
220
255
|
)
|
|
221
256
|
|
|
222
|
-
def ioassign_create(
|
|
223
|
-
|
|
257
|
+
def ioassign_create(
|
|
258
|
+
self,
|
|
259
|
+
ioassign="ioassign.x",
|
|
260
|
+
npool=1,
|
|
261
|
+
layout="ecma",
|
|
262
|
+
dbpath=None,
|
|
263
|
+
iocreate_path=None,
|
|
264
|
+
):
|
|
224
265
|
"""Build IO-Assign table."""
|
|
225
266
|
layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
|
|
226
267
|
if iocreate_path is None:
|
|
@@ -229,17 +270,28 @@ class OdbDriver:
|
|
|
229
270
|
self.sh.xperm(ioassign, force=True)
|
|
230
271
|
self.sh.mkdir(dbpath)
|
|
231
272
|
with self.env.clone() as lenv:
|
|
232
|
-
lenv[
|
|
273
|
+
lenv["ODB_IOASSIGN_BINARY"] = ioassign
|
|
233
274
|
self.fix_db_path(layout, dbpath, env=lenv)
|
|
234
|
-
self.sh.spawn(
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
275
|
+
self.sh.spawn(
|
|
276
|
+
[
|
|
277
|
+
iocreate_path,
|
|
278
|
+
"-d" + dbpath,
|
|
279
|
+
"-l" + layout,
|
|
280
|
+
"-n" + str(npool),
|
|
281
|
+
],
|
|
282
|
+
output=False,
|
|
283
|
+
)
|
|
239
284
|
return dbpath
|
|
240
285
|
|
|
241
|
-
def ioassign_merge(
|
|
242
|
-
|
|
286
|
+
def ioassign_merge(
|
|
287
|
+
self,
|
|
288
|
+
ioassign="ioassign.x",
|
|
289
|
+
layout="ecma",
|
|
290
|
+
odbnames=None,
|
|
291
|
+
dbpath=None,
|
|
292
|
+
iomerge_path=None,
|
|
293
|
+
iocreate_path=None,
|
|
294
|
+
):
|
|
243
295
|
"""Build IO-Assign table."""
|
|
244
296
|
layout, dbpath, thispwd = self._process_layout_dbpath(layout, dbpath)
|
|
245
297
|
if iomerge_path is None:
|
|
@@ -250,35 +302,39 @@ class OdbDriver:
|
|
|
250
302
|
ioassign = self.sh.path.abspath(ioassign)
|
|
251
303
|
self.sh.xperm(ioassign, force=True)
|
|
252
304
|
with self.sh.cdcontext(dbpath, create=True):
|
|
253
|
-
iocmd.extend([
|
|
305
|
+
iocmd.extend(["-d", thispwd])
|
|
254
306
|
for dbname in odbnames:
|
|
255
|
-
iocmd.extend([
|
|
307
|
+
iocmd.extend(["-t", dbname])
|
|
256
308
|
with self.env.clone() as lenv:
|
|
257
|
-
lenv[
|
|
258
|
-
if
|
|
259
|
-
lenv[
|
|
309
|
+
lenv["ODB_IOASSIGN_BINARY"] = ioassign
|
|
310
|
+
if "ODB_IOCREATE_COMMAND" not in lenv:
|
|
311
|
+
lenv["ODB_IOCREATE_COMMAND"] = iocreate_path
|
|
260
312
|
self.fix_db_path(layout, dbpath, env=lenv)
|
|
261
313
|
self.sh.spawn(iocmd, output=False)
|
|
262
314
|
return dbpath
|
|
263
315
|
|
|
264
316
|
def _ioassign_process(self, dbpaths, wmode):
|
|
265
|
-
with open(
|
|
317
|
+
with open("IOASSIGN", wmode) as fhgather:
|
|
266
318
|
for dbpath in dbpaths:
|
|
267
|
-
with open(self.sh.path.join(dbpath,
|
|
319
|
+
with open(self.sh.path.join(dbpath, "IOASSIGN")) as fhlay:
|
|
268
320
|
for line in fhlay:
|
|
269
321
|
fhgather.write(line)
|
|
270
322
|
|
|
271
323
|
def ioassign_gather(self, *dbpaths):
|
|
272
324
|
"""Gather IOASSIGN data from **dbpaths** databases and create a global IOASSIGN file."""
|
|
273
|
-
logger.info(
|
|
274
|
-
|
|
275
|
-
|
|
325
|
+
logger.info(
|
|
326
|
+
"ODB: creating a global IOASSIGN file from: %s",
|
|
327
|
+
",".join([self.sh.path.basename(db) for db in dbpaths]),
|
|
328
|
+
)
|
|
329
|
+
self._ioassign_process(dbpaths, "w")
|
|
276
330
|
|
|
277
331
|
def ioassign_append(self, *dbpaths):
|
|
278
332
|
"""Append IOASSIGN data from **dbpaths** databases into the global IOASSIGN file."""
|
|
279
|
-
logger.info(
|
|
280
|
-
|
|
281
|
-
|
|
333
|
+
logger.info(
|
|
334
|
+
"ODB: extending the IOASSIGN file with: %s",
|
|
335
|
+
",".join([self.sh.path.basename(db) for db in dbpaths]),
|
|
336
|
+
)
|
|
337
|
+
self._ioassign_process(dbpaths, "a")
|
|
282
338
|
|
|
283
339
|
def shuffle_setup(self, slots, mergedirect=False, ccmadirect=False):
|
|
284
340
|
"""Setup environment variables to control ODB shuffle behaviour.
|
|
@@ -286,8 +342,11 @@ class OdbDriver:
|
|
|
286
342
|
:param bool mergedirect: Run the shuffle procedure on the input database.
|
|
287
343
|
:param bool ccmadirect: Create a CCMA database at the end of the run.
|
|
288
344
|
"""
|
|
289
|
-
logger.info(
|
|
290
|
-
|
|
345
|
+
logger.info(
|
|
346
|
+
"ODB: shuffle_setup: mergedirect=%s, ccmadirect=%s",
|
|
347
|
+
str(mergedirect),
|
|
348
|
+
str(ccmadirect),
|
|
349
|
+
)
|
|
291
350
|
if mergedirect or ccmadirect:
|
|
292
351
|
self.env.update(
|
|
293
352
|
ODB_CCMA_TSLOTS=slots.nslot,
|
|
@@ -304,38 +363,63 @@ class OdbDriver:
|
|
|
304
363
|
def create_poolmask(self, layout, dbpath=None):
|
|
305
364
|
"""Request the poolmask file creation."""
|
|
306
365
|
layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
|
|
307
|
-
logger.info(
|
|
366
|
+
logger.info(
|
|
367
|
+
"ODB: requesting poolmask file for: %s (layout=%s).",
|
|
368
|
+
dbpath,
|
|
369
|
+
layout,
|
|
370
|
+
)
|
|
308
371
|
self.env.update(
|
|
309
372
|
ODB_CCMA_CREATE_POOLMASK=1,
|
|
310
|
-
ODB_CCMA_POOLMASK_FILE=self.sh.path.join(
|
|
373
|
+
ODB_CCMA_POOLMASK_FILE=self.sh.path.join(
|
|
374
|
+
dbpath, layout + ".poolmask"
|
|
375
|
+
),
|
|
311
376
|
)
|
|
312
377
|
|
|
313
378
|
def change_layout(self, layout, layout_new, dbpath=None):
|
|
314
379
|
"""Make the appropriate renaming of files in ECMA to CCMA."""
|
|
315
380
|
layout, dbpath, _ = self._process_layout_dbpath(layout, dbpath)
|
|
316
381
|
layout_new = layout_new.upper()
|
|
317
|
-
logger.info(
|
|
382
|
+
logger.info(
|
|
383
|
+
"ODB: changing layout (%s -> %s) for %s.",
|
|
384
|
+
layout,
|
|
385
|
+
layout_new,
|
|
386
|
+
dbpath,
|
|
387
|
+
)
|
|
318
388
|
to_cleanup = set()
|
|
319
389
|
for f in self.sh.ls(dbpath):
|
|
320
390
|
if self.sh.path.islink(self.sh.path.join(dbpath, f)):
|
|
321
391
|
fullpath = self.sh.path.join(dbpath, f)
|
|
322
392
|
target = self.sh.readlink(fullpath)
|
|
323
393
|
self.sh.unlink(fullpath)
|
|
324
|
-
self.sh.symlink(
|
|
325
|
-
|
|
394
|
+
self.sh.symlink(
|
|
395
|
+
target.replace(layout, layout_new),
|
|
396
|
+
fullpath.replace(layout, layout_new),
|
|
397
|
+
)
|
|
326
398
|
continue
|
|
327
|
-
if f in [n.format(layout) for n in (
|
|
328
|
-
self.sh.mv(
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
399
|
+
if f in [n.format(layout) for n in ("{:s}.dd", "{:s}.flags")]:
|
|
400
|
+
self.sh.mv(
|
|
401
|
+
self.sh.path.join(dbpath, f),
|
|
402
|
+
self.sh.path.join(dbpath, f.replace(layout, layout_new)),
|
|
403
|
+
)
|
|
404
|
+
if f in [
|
|
405
|
+
n.format(layout)
|
|
406
|
+
for n in (
|
|
407
|
+
"{:s}.iomap",
|
|
408
|
+
"{:s}.sch",
|
|
409
|
+
"{:s}.IOASSIGN",
|
|
410
|
+
"IOASSIGN.{:s}",
|
|
411
|
+
"IOASSIGN",
|
|
412
|
+
)
|
|
413
|
+
]:
|
|
414
|
+
tmp_target = self.sh.path.join(dbpath, f + ".tmp_new")
|
|
333
415
|
with open(self.sh.path.join(dbpath, f)) as inodb:
|
|
334
|
-
with open(tmp_target,
|
|
416
|
+
with open(tmp_target, "w") as outodb:
|
|
335
417
|
for line in inodb:
|
|
336
418
|
outodb.write(line.replace(layout, layout_new))
|
|
337
|
-
self.sh.mv(
|
|
338
|
-
|
|
419
|
+
self.sh.mv(
|
|
420
|
+
tmp_target,
|
|
421
|
+
self.sh.path.join(dbpath, f.replace(layout, layout_new)),
|
|
422
|
+
)
|
|
339
423
|
if layout in f:
|
|
340
424
|
to_cleanup.add(self.sh.path.join(dbpath, f))
|
|
341
425
|
for f_name in to_cleanup:
|
|
@@ -347,44 +431,45 @@ odbmix_attributes = footprints.Footprint(
|
|
|
347
431
|
info="Abstract ODB footprints' attributes.",
|
|
348
432
|
attr=dict(
|
|
349
433
|
npool=dict(
|
|
350
|
-
info=
|
|
434
|
+
info="The number of pool(s) in the ODB database.",
|
|
351
435
|
type=int,
|
|
352
436
|
optional=True,
|
|
353
437
|
default=1,
|
|
354
438
|
),
|
|
355
439
|
iomethod=dict(
|
|
356
|
-
info=
|
|
440
|
+
info="The io_method of the ODB database.",
|
|
357
441
|
type=int,
|
|
358
442
|
optional=True,
|
|
359
443
|
default=1,
|
|
360
444
|
doc_zorder=-50,
|
|
361
445
|
),
|
|
362
446
|
slots=dict(
|
|
363
|
-
info=
|
|
447
|
+
info="The timeslots of the assimilation window.",
|
|
364
448
|
type=TimeSlots,
|
|
365
449
|
optional=True,
|
|
366
|
-
default=TimeSlots(7, chunk=
|
|
450
|
+
default=TimeSlots(7, chunk="PT1H"),
|
|
367
451
|
),
|
|
368
452
|
virtualdb=dict(
|
|
369
|
-
info=
|
|
453
|
+
info="The type of the virtual ODB database.",
|
|
370
454
|
optional=True,
|
|
371
|
-
default=
|
|
372
|
-
access=
|
|
455
|
+
default="ecma",
|
|
456
|
+
access="rwx",
|
|
373
457
|
doc_visibility=footprints.doc.visibility.ADVANCED,
|
|
374
458
|
),
|
|
375
459
|
date=dict(
|
|
376
|
-
info=
|
|
460
|
+
info="The current run date.",
|
|
377
461
|
optional=True,
|
|
378
|
-
access=
|
|
462
|
+
access="rwx",
|
|
379
463
|
type=bdate.Date,
|
|
380
464
|
doc_zorder=-50,
|
|
381
465
|
),
|
|
382
466
|
ioassign=dict(
|
|
383
|
-
info=
|
|
467
|
+
info="The path to the ioassign binary (needed for merge/create actions",
|
|
384
468
|
optional=True,
|
|
385
|
-
default=
|
|
386
|
-
)
|
|
387
|
-
)
|
|
469
|
+
default="ioassign.x",
|
|
470
|
+
),
|
|
471
|
+
),
|
|
472
|
+
)
|
|
388
473
|
|
|
389
474
|
|
|
390
475
|
@algo_component_deco_mixin_autodoc
|
|
@@ -402,12 +487,14 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
402
487
|
manually if needed.
|
|
403
488
|
"""
|
|
404
489
|
|
|
405
|
-
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
490
|
+
_MIXIN_EXTRA_FOOTPRINTS = [
|
|
491
|
+
odbmix_attributes,
|
|
492
|
+
]
|
|
406
493
|
|
|
407
494
|
def _odbobj_init(self, rh, opts): # @UnusedVariable
|
|
408
495
|
"""Setup the OdbDriver object."""
|
|
409
|
-
cycle = ArpIfsSimplifiedCycle(
|
|
410
|
-
if rh and hasattr(rh.resource,
|
|
496
|
+
cycle = ArpIfsSimplifiedCycle("cy01")
|
|
497
|
+
if rh and hasattr(rh.resource, "cycle"):
|
|
411
498
|
cycle = rh.resource.cycle
|
|
412
499
|
self._odb = OdbDriver(
|
|
413
500
|
cycle=cycle,
|
|
@@ -436,20 +523,23 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
436
523
|
@property
|
|
437
524
|
def odb(self):
|
|
438
525
|
"""Access to a properly initialised :class:`OdbDriver` object."""
|
|
439
|
-
if not hasattr(self,
|
|
440
|
-
raise RuntimeError(
|
|
526
|
+
if not hasattr(self, "_odb"):
|
|
527
|
+
raise RuntimeError("Uninitialised *odb* object.")
|
|
441
528
|
return self._odb
|
|
442
529
|
|
|
443
530
|
def lookupodb(self, fatal=True):
|
|
444
531
|
"""Return a list of effective input resources which are odb observations."""
|
|
445
532
|
allodb = [
|
|
446
|
-
x
|
|
447
|
-
|
|
533
|
+
x
|
|
534
|
+
for x in self.context.sequence.effective_inputs(
|
|
535
|
+
kind="observations"
|
|
536
|
+
)
|
|
537
|
+
if x.rh.container.actualfmt == "odb"
|
|
448
538
|
]
|
|
449
539
|
allodb.sort(key=lambda s: s.rh.resource.part)
|
|
450
540
|
if not allodb and fatal:
|
|
451
|
-
logger.critical(
|
|
452
|
-
raise ValueError(
|
|
541
|
+
logger.critical("Missing ODB input data for %s", self.fullname())
|
|
542
|
+
raise ValueError("Missing ODB input data")
|
|
453
543
|
return allodb
|
|
454
544
|
|
|
455
545
|
def odb_date_and_layout_from_sections(self, odbsections):
|
|
@@ -465,23 +555,32 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
465
555
|
raise AlgoComponentError("Inconsistent ODB dates")
|
|
466
556
|
self.virtualdb = alllayouts.pop()
|
|
467
557
|
self.date = alldates.pop()
|
|
468
|
-
logger.info(
|
|
469
|
-
|
|
558
|
+
logger.info(
|
|
559
|
+
"ODB: Detected from ODB database(s). self.date=%s, self.virtualdb=%s.",
|
|
560
|
+
self.date.stdvortex,
|
|
561
|
+
self.virtualdb,
|
|
562
|
+
)
|
|
470
563
|
|
|
471
564
|
def _odb_find_ioassign_script(self, purpose):
|
|
472
565
|
"""Look for ioassign script of *purpose" attribute, return path."""
|
|
473
|
-
scripts = [
|
|
474
|
-
|
|
475
|
-
|
|
566
|
+
scripts = [
|
|
567
|
+
x.rh.container.abspath
|
|
568
|
+
for x in self.context.sequence.effective_inputs(
|
|
569
|
+
kind="ioassign_script"
|
|
570
|
+
)
|
|
571
|
+
if x.rh.resource.purpose == purpose
|
|
572
|
+
]
|
|
476
573
|
if len(scripts) > 1:
|
|
477
|
-
raise AlgoComponentError(
|
|
574
|
+
raise AlgoComponentError(
|
|
575
|
+
"More than one purpose={} ioassign_script found in resources."
|
|
576
|
+
)
|
|
478
577
|
elif len(scripts) == 1:
|
|
479
578
|
self.system.xperm(scripts[0], force=True)
|
|
480
579
|
return scripts[0]
|
|
481
580
|
else:
|
|
482
581
|
return None
|
|
483
582
|
|
|
484
|
-
def odb_merge_if_needed(self, odbsections, subdir=
|
|
583
|
+
def odb_merge_if_needed(self, odbsections, subdir="."):
|
|
485
584
|
"""
|
|
486
585
|
If multiple ODB databases are listed in the **odsections** section list,
|
|
487
586
|
start an ODB merge.
|
|
@@ -489,10 +588,10 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
489
588
|
:return: The path to the ODB database (the virtual database if a merge is
|
|
490
589
|
performed).
|
|
491
590
|
"""
|
|
492
|
-
if len(odbsections) > 1 or self.virtualdb.lower() ==
|
|
493
|
-
logger.info(
|
|
494
|
-
iomerge_p = self._odb_find_ioassign_script(
|
|
495
|
-
iocreate_p = self._odb_find_ioassign_script(
|
|
591
|
+
if len(odbsections) > 1 or self.virtualdb.lower() == "ecma":
|
|
592
|
+
logger.info("ODB: merge for: %s.", self.virtualdb)
|
|
593
|
+
iomerge_p = self._odb_find_ioassign_script("merge")
|
|
594
|
+
iocreate_p = self._odb_find_ioassign_script("create")
|
|
496
595
|
with self.system.cdcontext(subdir):
|
|
497
596
|
virtualdb_path = self.odb.ioassign_merge(
|
|
498
597
|
layout=self.virtualdb,
|
|
@@ -502,7 +601,9 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
502
601
|
iocreate_path=iocreate_p,
|
|
503
602
|
)
|
|
504
603
|
else:
|
|
505
|
-
virtualdb_path = self.system.path.abspath(
|
|
604
|
+
virtualdb_path = self.system.path.abspath(
|
|
605
|
+
odbsections[0].rh.container.localpath()
|
|
606
|
+
)
|
|
506
607
|
return virtualdb_path
|
|
507
608
|
|
|
508
609
|
def odb_create_db(self, layout, dbpath=None):
|
|
@@ -517,28 +618,36 @@ class OdbComponentDecoMixin(AlgoComponentDecoMixin):
|
|
|
517
618
|
npool=self.npool,
|
|
518
619
|
ioassign=self._x_ioassign,
|
|
519
620
|
dbpath=dbpath,
|
|
520
|
-
iocreate_path=self._odb_find_ioassign_script(
|
|
621
|
+
iocreate_path=self._odb_find_ioassign_script("create"),
|
|
521
622
|
)
|
|
522
|
-
logger.info(
|
|
623
|
+
logger.info("ODB: database created: %s (layout=%s).", dbout, layout)
|
|
523
624
|
return dbout
|
|
524
625
|
|
|
525
626
|
def odb_handle_raw_dbs(self):
|
|
526
627
|
"""Look for extras ODB raw databases and fix the environment accordingly."""
|
|
527
628
|
odbraw = [
|
|
528
|
-
x.rh
|
|
529
|
-
|
|
629
|
+
x.rh
|
|
630
|
+
for x in self.context.sequence.effective_inputs(kind="odbraw")
|
|
631
|
+
if x.rh.container.actualfmt == "odb"
|
|
530
632
|
]
|
|
531
633
|
if not odbraw:
|
|
532
|
-
logger.error(
|
|
634
|
+
logger.error("No ODB raw databases found.")
|
|
533
635
|
else:
|
|
534
636
|
for rraw in odbraw:
|
|
535
637
|
rawpath = rraw.container.localpath()
|
|
536
638
|
self.odb.fix_db_path(rraw.resource.layout, rawpath)
|
|
537
|
-
for badlink in [
|
|
538
|
-
|
|
539
|
-
|
|
639
|
+
for badlink in [
|
|
640
|
+
bl
|
|
641
|
+
for bl in self.system.glob(
|
|
642
|
+
self.system.path.join(rawpath, "*.h")
|
|
643
|
+
)
|
|
644
|
+
if self.system.path.islink(bl)
|
|
645
|
+
and not self.system.path.exists(bl)
|
|
646
|
+
]:
|
|
540
647
|
self.system.unlink(badlink)
|
|
541
|
-
self.odb.ioassign_append(
|
|
648
|
+
self.odb.ioassign_append(
|
|
649
|
+
*[rraw.container.localpath() for rraw in odbraw]
|
|
650
|
+
)
|
|
542
651
|
return odbraw
|
|
543
652
|
|
|
544
653
|
def odb_rw_or_overwrite_method(self, *dbsections):
|