vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +75 -47
- vortex/algo/__init__.py +3 -2
- vortex/algo/components.py +944 -618
- vortex/algo/mpitools.py +802 -497
- vortex/algo/mpitools_templates/__init__.py +1 -0
- vortex/algo/serversynctools.py +34 -33
- vortex/config.py +19 -22
- vortex/data/__init__.py +9 -3
- vortex/data/abstractstores.py +593 -655
- vortex/data/containers.py +217 -162
- vortex/data/contents.py +65 -39
- vortex/data/executables.py +93 -102
- vortex/data/flow.py +40 -34
- vortex/data/geometries.py +228 -132
- vortex/data/handlers.py +436 -227
- vortex/data/outflow.py +15 -15
- vortex/data/providers.py +185 -163
- vortex/data/resources.py +48 -42
- vortex/data/stores.py +540 -417
- vortex/data/sync_templates/__init__.py +0 -0
- vortex/gloves.py +114 -87
- vortex/layout/__init__.py +1 -8
- vortex/layout/contexts.py +150 -84
- vortex/layout/dataflow.py +353 -202
- vortex/layout/monitor.py +264 -128
- vortex/nwp/__init__.py +5 -2
- vortex/nwp/algo/__init__.py +14 -5
- vortex/nwp/algo/assim.py +205 -151
- vortex/nwp/algo/clim.py +683 -517
- vortex/nwp/algo/coupling.py +447 -225
- vortex/nwp/algo/eda.py +437 -229
- vortex/nwp/algo/eps.py +403 -231
- vortex/nwp/algo/forecasts.py +416 -275
- vortex/nwp/algo/fpserver.py +683 -307
- vortex/nwp/algo/ifsnaming.py +205 -145
- vortex/nwp/algo/ifsroot.py +215 -122
- vortex/nwp/algo/monitoring.py +137 -76
- vortex/nwp/algo/mpitools.py +330 -190
- vortex/nwp/algo/odbtools.py +637 -353
- vortex/nwp/algo/oopsroot.py +454 -273
- vortex/nwp/algo/oopstests.py +90 -56
- vortex/nwp/algo/request.py +287 -206
- vortex/nwp/algo/stdpost.py +878 -522
- vortex/nwp/data/__init__.py +22 -4
- vortex/nwp/data/assim.py +125 -137
- vortex/nwp/data/boundaries.py +121 -68
- vortex/nwp/data/climfiles.py +193 -211
- vortex/nwp/data/configfiles.py +73 -69
- vortex/nwp/data/consts.py +426 -401
- vortex/nwp/data/ctpini.py +59 -43
- vortex/nwp/data/diagnostics.py +94 -66
- vortex/nwp/data/eda.py +50 -51
- vortex/nwp/data/eps.py +195 -146
- vortex/nwp/data/executables.py +440 -434
- vortex/nwp/data/fields.py +63 -48
- vortex/nwp/data/gridfiles.py +183 -111
- vortex/nwp/data/logs.py +250 -217
- vortex/nwp/data/modelstates.py +180 -151
- vortex/nwp/data/monitoring.py +72 -99
- vortex/nwp/data/namelists.py +254 -202
- vortex/nwp/data/obs.py +400 -308
- vortex/nwp/data/oopsexec.py +22 -20
- vortex/nwp/data/providers.py +90 -65
- vortex/nwp/data/query.py +71 -82
- vortex/nwp/data/stores.py +49 -36
- vortex/nwp/data/surfex.py +136 -137
- vortex/nwp/syntax/__init__.py +1 -1
- vortex/nwp/syntax/stdattrs.py +173 -111
- vortex/nwp/tools/__init__.py +2 -2
- vortex/nwp/tools/addons.py +22 -17
- vortex/nwp/tools/agt.py +24 -12
- vortex/nwp/tools/bdap.py +16 -5
- vortex/nwp/tools/bdcp.py +4 -1
- vortex/nwp/tools/bdm.py +3 -0
- vortex/nwp/tools/bdmp.py +14 -9
- vortex/nwp/tools/conftools.py +728 -378
- vortex/nwp/tools/drhook.py +12 -8
- vortex/nwp/tools/grib.py +65 -39
- vortex/nwp/tools/gribdiff.py +22 -17
- vortex/nwp/tools/ifstools.py +82 -42
- vortex/nwp/tools/igastuff.py +167 -143
- vortex/nwp/tools/mars.py +14 -2
- vortex/nwp/tools/odb.py +234 -125
- vortex/nwp/tools/partitioning.py +61 -37
- vortex/nwp/tools/satrad.py +27 -12
- vortex/nwp/util/async.py +83 -55
- vortex/nwp/util/beacon.py +10 -10
- vortex/nwp/util/diffpygram.py +174 -86
- vortex/nwp/util/ens.py +144 -63
- vortex/nwp/util/hooks.py +30 -19
- vortex/nwp/util/taskdeco.py +28 -24
- vortex/nwp/util/usepygram.py +278 -172
- vortex/nwp/util/usetnt.py +31 -17
- vortex/sessions.py +72 -39
- vortex/syntax/__init__.py +1 -1
- vortex/syntax/stdattrs.py +410 -171
- vortex/syntax/stddeco.py +31 -22
- vortex/toolbox.py +327 -192
- vortex/tools/__init__.py +11 -2
- vortex/tools/actions.py +110 -121
- vortex/tools/addons.py +111 -92
- vortex/tools/arm.py +42 -22
- vortex/tools/compression.py +72 -69
- vortex/tools/date.py +11 -4
- vortex/tools/delayedactions.py +242 -132
- vortex/tools/env.py +75 -47
- vortex/tools/folder.py +342 -171
- vortex/tools/grib.py +341 -162
- vortex/tools/lfi.py +423 -216
- vortex/tools/listings.py +109 -40
- vortex/tools/names.py +218 -156
- vortex/tools/net.py +655 -299
- vortex/tools/parallelism.py +93 -61
- vortex/tools/prestaging.py +55 -31
- vortex/tools/schedulers.py +172 -105
- vortex/tools/services.py +403 -334
- vortex/tools/storage.py +293 -358
- vortex/tools/surfex.py +24 -24
- vortex/tools/systems.py +1234 -643
- vortex/tools/targets.py +156 -100
- vortex/util/__init__.py +1 -1
- vortex/util/config.py +378 -327
- vortex/util/empty.py +2 -2
- vortex/util/helpers.py +56 -24
- vortex/util/introspection.py +18 -12
- vortex/util/iosponge.py +8 -4
- vortex/util/roles.py +4 -6
- vortex/util/storefunctions.py +39 -13
- vortex/util/structs.py +3 -3
- vortex/util/worker.py +29 -17
- vortex_nwp-2.1.0.dist-info/METADATA +67 -0
- vortex_nwp-2.1.0.dist-info/RECORD +144 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
- vortex/layout/appconf.py +0 -109
- vortex/layout/jobs.py +0 -1276
- vortex/layout/nodes.py +0 -1424
- vortex/layout/subjobs.py +0 -464
- vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
- vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
- {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Empty init file
|
vortex/algo/serversynctools.py
CHANGED
|
@@ -23,30 +23,25 @@ class ServerSyncTool(footprints.FootprintBase):
|
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
_abstract = True
|
|
26
|
-
_collector = (
|
|
26
|
+
_collector = ("serversynctool",)
|
|
27
27
|
_footprint = dict(
|
|
28
|
-
info=
|
|
28
|
+
info="Abstract Server Synchronisation Tool",
|
|
29
29
|
attr=dict(
|
|
30
|
-
method=dict(
|
|
31
|
-
),
|
|
30
|
+
method=dict(),
|
|
32
31
|
medium=dict(
|
|
33
32
|
optional=True,
|
|
34
33
|
),
|
|
35
|
-
raiseonexit=dict(
|
|
36
|
-
type=bool,
|
|
37
|
-
optional=True,
|
|
38
|
-
default=True
|
|
39
|
-
),
|
|
34
|
+
raiseonexit=dict(type=bool, optional=True, default=True),
|
|
40
35
|
checkinterval=dict(
|
|
41
36
|
type=int,
|
|
42
37
|
optional=True,
|
|
43
38
|
default=10,
|
|
44
39
|
),
|
|
45
|
-
)
|
|
40
|
+
),
|
|
46
41
|
)
|
|
47
42
|
|
|
48
43
|
def __init__(self, *args, **kw):
|
|
49
|
-
logger.debug(
|
|
44
|
+
logger.debug("Server Synchronisation Tool init %s", self.__class__)
|
|
50
45
|
self._check_callback = lambda: True
|
|
51
46
|
super().__init__(*args, **kw)
|
|
52
47
|
|
|
@@ -80,19 +75,19 @@ class ServerSyncSimpleSocket(ServerSyncTool):
|
|
|
80
75
|
"""
|
|
81
76
|
|
|
82
77
|
_footprint = dict(
|
|
83
|
-
info=
|
|
78
|
+
info="Server Synchronisation Tool that uses a Socket",
|
|
84
79
|
attr=dict(
|
|
85
80
|
method=dict(
|
|
86
|
-
values=[
|
|
81
|
+
values=["simple_socket"],
|
|
87
82
|
),
|
|
88
83
|
medium=dict(
|
|
89
84
|
optional=False,
|
|
90
85
|
),
|
|
91
86
|
tplname=dict(
|
|
92
87
|
optional=True,
|
|
93
|
-
default=
|
|
88
|
+
default="@servsync-simplesocket.tpl",
|
|
94
89
|
),
|
|
95
|
-
)
|
|
90
|
+
),
|
|
96
91
|
)
|
|
97
92
|
|
|
98
93
|
def __init__(self, *args, **kw):
|
|
@@ -102,7 +97,7 @@ class ServerSyncSimpleSocket(ServerSyncTool):
|
|
|
102
97
|
try:
|
|
103
98
|
self._socket.bind((socket.getfqdn(), 0))
|
|
104
99
|
except OSError:
|
|
105
|
-
self._socket.bind((
|
|
100
|
+
self._socket.bind(("localhost", 0))
|
|
106
101
|
self._socket.settimeout(self.checkinterval)
|
|
107
102
|
self._socket.listen(1)
|
|
108
103
|
# Current connection
|
|
@@ -110,11 +105,13 @@ class ServerSyncSimpleSocket(ServerSyncTool):
|
|
|
110
105
|
# Create the script that will be called by the server
|
|
111
106
|
t = sessions.current()
|
|
112
107
|
tpl = config.load_template(t, self.tplname)
|
|
113
|
-
with open(self.medium,
|
|
114
|
-
fd.write(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
108
|
+
with open(self.medium, "w") as fd:
|
|
109
|
+
fd.write(
|
|
110
|
+
tpl.substitute(
|
|
111
|
+
python=sys.executable,
|
|
112
|
+
address=self._socket.getsockname(),
|
|
113
|
+
)
|
|
114
|
+
)
|
|
118
115
|
t.sh.chmod(self.medium, 0o555)
|
|
119
116
|
|
|
120
117
|
def __del__(self):
|
|
@@ -130,13 +127,13 @@ class ServerSyncSimpleSocket(ServerSyncTool):
|
|
|
130
127
|
if self._socket_conn is not None:
|
|
131
128
|
logger.info('Sending "%s" to the server.', mess)
|
|
132
129
|
# NB: For send/recv, the settimeout also applies...
|
|
133
|
-
self._socket_conn.send(mess.encode(encoding=
|
|
134
|
-
repl = self._socket_conn.recv(255).decode(encoding=
|
|
130
|
+
self._socket_conn.send(mess.encode(encoding="utf-8"))
|
|
131
|
+
repl = self._socket_conn.recv(255).decode(encoding="utf-8")
|
|
135
132
|
logger.info('Server replied "%s" to %s.', repl, mess)
|
|
136
133
|
self._socket_conn.close()
|
|
137
134
|
self._socket_conn = None
|
|
138
|
-
if repl !=
|
|
139
|
-
raise ValueError(mess +
|
|
135
|
+
if repl != "OK":
|
|
136
|
+
raise ValueError(mess + " failed")
|
|
140
137
|
return True
|
|
141
138
|
else:
|
|
142
139
|
# This should not happen ! If we are sitting here, it's most likely
|
|
@@ -144,27 +141,31 @@ class ServerSyncSimpleSocket(ServerSyncTool):
|
|
|
144
141
|
return False
|
|
145
142
|
|
|
146
143
|
def trigger_wait(self):
|
|
147
|
-
logger.info(
|
|
144
|
+
logger.info("Waiting for the server to complete")
|
|
148
145
|
while self._socket_conn is None and self._check_callback():
|
|
149
146
|
try:
|
|
150
|
-
self._socket_conn, addr =
|
|
147
|
+
self._socket_conn, addr = (
|
|
148
|
+
self._socket.accept()
|
|
149
|
+
) # @UnusedVariable
|
|
151
150
|
except socket.timeout:
|
|
152
|
-
logger.debug(
|
|
151
|
+
logger.debug(
|
|
152
|
+
"Socket accept timed-out: checking for the server..."
|
|
153
|
+
)
|
|
153
154
|
self._socket_conn = None
|
|
154
155
|
if self._socket_conn is None:
|
|
155
156
|
if self.raiseonexit:
|
|
156
|
-
raise OSError(
|
|
157
|
+
raise OSError("Apparently the server died.")
|
|
157
158
|
else:
|
|
158
|
-
logger.info(
|
|
159
|
+
logger.info("The server stopped.")
|
|
159
160
|
else:
|
|
160
161
|
self._socket_conn.settimeout(self.checkinterval)
|
|
161
|
-
logger.info(
|
|
162
|
+
logger.info("The server is now waiting")
|
|
162
163
|
|
|
163
164
|
def trigger_run(self):
|
|
164
165
|
# Tell the server that everything is ready
|
|
165
|
-
self._command(
|
|
166
|
+
self._command("STEP")
|
|
166
167
|
# Wait for the server to complete its work
|
|
167
168
|
self.trigger_wait()
|
|
168
169
|
|
|
169
170
|
def trigger_stop(self):
|
|
170
|
-
return self._command(
|
|
171
|
+
return self._command("STOP")
|
vortex/config.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
the value of configuration options, respectively.
|
|
3
3
|
|
|
4
4
|
"""
|
|
5
|
+
|
|
5
6
|
import tomli
|
|
6
7
|
|
|
7
8
|
from bronx.fancies import loggers
|
|
@@ -19,6 +20,10 @@ VORTEX_CONFIG = {}
|
|
|
19
20
|
logger = loggers.getLogger(__name__)
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
class ConfigurationError(Exception):
|
|
24
|
+
"""Something is wrong with the provided configuration"""
|
|
25
|
+
|
|
26
|
+
|
|
22
27
|
def load_config(configpath="vortex.toml"):
|
|
23
28
|
"""Load configuration from a TOML configuration file
|
|
24
29
|
|
|
@@ -38,21 +43,15 @@ def load_config(configpath="vortex.toml"):
|
|
|
38
43
|
global VORTEX_CONFIG
|
|
39
44
|
try:
|
|
40
45
|
with open(configpath, "rb") as f:
|
|
41
|
-
|
|
46
|
+
VORTEX_CONFIG = tomli.load(f)
|
|
42
47
|
print(f"Successfully read configuration file {configpath}")
|
|
43
48
|
except FileNotFoundError:
|
|
44
|
-
print(
|
|
45
|
-
|
|
46
|
-
" (not found)."
|
|
47
|
-
)
|
|
48
|
-
print(
|
|
49
|
-
"Use load_config(/path/to/config) to update the configuration"
|
|
50
|
-
)
|
|
49
|
+
print(f"Could not read configuration file {configpath} (not found).")
|
|
50
|
+
print("Use load_config(/path/to/config) to update the configuration")
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
def print_config():
|
|
54
|
-
"""Print configuration (key, value) pairs
|
|
55
|
-
"""
|
|
54
|
+
"""Print configuration (key, value) pairs"""
|
|
56
55
|
if VORTEX_CONFIG:
|
|
57
56
|
for k, v in VORTEX_CONFIG:
|
|
58
57
|
print(k.upper(), v)
|
|
@@ -66,18 +65,19 @@ def from_config(section, key=None):
|
|
|
66
65
|
"""
|
|
67
66
|
try:
|
|
68
67
|
subconfig = VORTEX_CONFIG[section]
|
|
69
|
-
except KeyError
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
68
|
+
except KeyError:
|
|
69
|
+
raise ConfigurationError(
|
|
70
|
+
f"Missing configuration section {section}",
|
|
71
|
+
)
|
|
73
72
|
if not key:
|
|
74
73
|
return subconfig
|
|
75
74
|
|
|
76
75
|
try:
|
|
77
76
|
value = subconfig[key]
|
|
78
|
-
except KeyError
|
|
79
|
-
|
|
80
|
-
|
|
77
|
+
except KeyError:
|
|
78
|
+
raise ConfigurationError(
|
|
79
|
+
f"Missing configuration key {key} in section {section}",
|
|
80
|
+
)
|
|
81
81
|
return value
|
|
82
82
|
|
|
83
83
|
|
|
@@ -87,9 +87,7 @@ def set_config(section, key, value):
|
|
|
87
87
|
if section not in VORTEX_CONFIG.keys():
|
|
88
88
|
VORTEX_CONFIG[section] = {}
|
|
89
89
|
if key in VORTEX_CONFIG[section]:
|
|
90
|
-
logger.warning(
|
|
91
|
-
f"Updating existing configuration {section}:{key}"
|
|
92
|
-
)
|
|
90
|
+
logger.warning(f"Updating existing configuration {section}:{key}")
|
|
93
91
|
VORTEX_CONFIG[section][key] = value
|
|
94
92
|
|
|
95
93
|
|
|
@@ -108,8 +106,7 @@ def is_defined(section, key=None):
|
|
|
108
106
|
|
|
109
107
|
|
|
110
108
|
def get_from_config_w_default(section, key, default):
|
|
111
|
-
logger.info(f"Reading config value {section}.{key}")
|
|
112
109
|
try:
|
|
113
110
|
return from_config(section, key)
|
|
114
|
-
except
|
|
111
|
+
except ConfigurationError:
|
|
115
112
|
return default
|
vortex/data/__init__.py
CHANGED
|
@@ -4,10 +4,16 @@ Abstract classes involved in data management within VORTEX.
|
|
|
4
4
|
Actual resources and custom providers should be defined in dedicated packages.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
from . import handlers
|
|
8
|
-
|
|
7
|
+
from . import handlers as handlers
|
|
8
|
+
from . import resources as resources
|
|
9
|
+
from . import containers as containers
|
|
10
|
+
from . import contents as contents
|
|
11
|
+
from . import providers as providers
|
|
12
|
+
from . import executables as executables
|
|
13
|
+
from . import stores as stores
|
|
14
|
+
from . import geometries as geometries
|
|
9
15
|
|
|
10
16
|
#: No automatic export
|
|
11
17
|
__all__ = []
|
|
12
18
|
|
|
13
|
-
__tocinfoline__ =
|
|
19
|
+
__tocinfoline__ = "Abstract classes involved in data management within VORTEX"
|