vortex-nwp 2.0.0b1__py3-none-any.whl → 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. vortex/__init__.py +75 -47
  2. vortex/algo/__init__.py +3 -2
  3. vortex/algo/components.py +944 -618
  4. vortex/algo/mpitools.py +802 -497
  5. vortex/algo/mpitools_templates/__init__.py +1 -0
  6. vortex/algo/serversynctools.py +34 -33
  7. vortex/config.py +19 -22
  8. vortex/data/__init__.py +9 -3
  9. vortex/data/abstractstores.py +593 -655
  10. vortex/data/containers.py +217 -162
  11. vortex/data/contents.py +65 -39
  12. vortex/data/executables.py +93 -102
  13. vortex/data/flow.py +40 -34
  14. vortex/data/geometries.py +228 -132
  15. vortex/data/handlers.py +436 -227
  16. vortex/data/outflow.py +15 -15
  17. vortex/data/providers.py +185 -163
  18. vortex/data/resources.py +48 -42
  19. vortex/data/stores.py +540 -417
  20. vortex/data/sync_templates/__init__.py +0 -0
  21. vortex/gloves.py +114 -87
  22. vortex/layout/__init__.py +1 -8
  23. vortex/layout/contexts.py +150 -84
  24. vortex/layout/dataflow.py +353 -202
  25. vortex/layout/monitor.py +264 -128
  26. vortex/nwp/__init__.py +5 -2
  27. vortex/nwp/algo/__init__.py +14 -5
  28. vortex/nwp/algo/assim.py +205 -151
  29. vortex/nwp/algo/clim.py +683 -517
  30. vortex/nwp/algo/coupling.py +447 -225
  31. vortex/nwp/algo/eda.py +437 -229
  32. vortex/nwp/algo/eps.py +403 -231
  33. vortex/nwp/algo/forecasts.py +416 -275
  34. vortex/nwp/algo/fpserver.py +683 -307
  35. vortex/nwp/algo/ifsnaming.py +205 -145
  36. vortex/nwp/algo/ifsroot.py +215 -122
  37. vortex/nwp/algo/monitoring.py +137 -76
  38. vortex/nwp/algo/mpitools.py +330 -190
  39. vortex/nwp/algo/odbtools.py +637 -353
  40. vortex/nwp/algo/oopsroot.py +454 -273
  41. vortex/nwp/algo/oopstests.py +90 -56
  42. vortex/nwp/algo/request.py +287 -206
  43. vortex/nwp/algo/stdpost.py +878 -522
  44. vortex/nwp/data/__init__.py +22 -4
  45. vortex/nwp/data/assim.py +125 -137
  46. vortex/nwp/data/boundaries.py +121 -68
  47. vortex/nwp/data/climfiles.py +193 -211
  48. vortex/nwp/data/configfiles.py +73 -69
  49. vortex/nwp/data/consts.py +426 -401
  50. vortex/nwp/data/ctpini.py +59 -43
  51. vortex/nwp/data/diagnostics.py +94 -66
  52. vortex/nwp/data/eda.py +50 -51
  53. vortex/nwp/data/eps.py +195 -146
  54. vortex/nwp/data/executables.py +440 -434
  55. vortex/nwp/data/fields.py +63 -48
  56. vortex/nwp/data/gridfiles.py +183 -111
  57. vortex/nwp/data/logs.py +250 -217
  58. vortex/nwp/data/modelstates.py +180 -151
  59. vortex/nwp/data/monitoring.py +72 -99
  60. vortex/nwp/data/namelists.py +254 -202
  61. vortex/nwp/data/obs.py +400 -308
  62. vortex/nwp/data/oopsexec.py +22 -20
  63. vortex/nwp/data/providers.py +90 -65
  64. vortex/nwp/data/query.py +71 -82
  65. vortex/nwp/data/stores.py +49 -36
  66. vortex/nwp/data/surfex.py +136 -137
  67. vortex/nwp/syntax/__init__.py +1 -1
  68. vortex/nwp/syntax/stdattrs.py +173 -111
  69. vortex/nwp/tools/__init__.py +2 -2
  70. vortex/nwp/tools/addons.py +22 -17
  71. vortex/nwp/tools/agt.py +24 -12
  72. vortex/nwp/tools/bdap.py +16 -5
  73. vortex/nwp/tools/bdcp.py +4 -1
  74. vortex/nwp/tools/bdm.py +3 -0
  75. vortex/nwp/tools/bdmp.py +14 -9
  76. vortex/nwp/tools/conftools.py +728 -378
  77. vortex/nwp/tools/drhook.py +12 -8
  78. vortex/nwp/tools/grib.py +65 -39
  79. vortex/nwp/tools/gribdiff.py +22 -17
  80. vortex/nwp/tools/ifstools.py +82 -42
  81. vortex/nwp/tools/igastuff.py +167 -143
  82. vortex/nwp/tools/mars.py +14 -2
  83. vortex/nwp/tools/odb.py +234 -125
  84. vortex/nwp/tools/partitioning.py +61 -37
  85. vortex/nwp/tools/satrad.py +27 -12
  86. vortex/nwp/util/async.py +83 -55
  87. vortex/nwp/util/beacon.py +10 -10
  88. vortex/nwp/util/diffpygram.py +174 -86
  89. vortex/nwp/util/ens.py +144 -63
  90. vortex/nwp/util/hooks.py +30 -19
  91. vortex/nwp/util/taskdeco.py +28 -24
  92. vortex/nwp/util/usepygram.py +278 -172
  93. vortex/nwp/util/usetnt.py +31 -17
  94. vortex/sessions.py +72 -39
  95. vortex/syntax/__init__.py +1 -1
  96. vortex/syntax/stdattrs.py +410 -171
  97. vortex/syntax/stddeco.py +31 -22
  98. vortex/toolbox.py +327 -192
  99. vortex/tools/__init__.py +11 -2
  100. vortex/tools/actions.py +110 -121
  101. vortex/tools/addons.py +111 -92
  102. vortex/tools/arm.py +42 -22
  103. vortex/tools/compression.py +72 -69
  104. vortex/tools/date.py +11 -4
  105. vortex/tools/delayedactions.py +242 -132
  106. vortex/tools/env.py +75 -47
  107. vortex/tools/folder.py +342 -171
  108. vortex/tools/grib.py +341 -162
  109. vortex/tools/lfi.py +423 -216
  110. vortex/tools/listings.py +109 -40
  111. vortex/tools/names.py +218 -156
  112. vortex/tools/net.py +655 -299
  113. vortex/tools/parallelism.py +93 -61
  114. vortex/tools/prestaging.py +55 -31
  115. vortex/tools/schedulers.py +172 -105
  116. vortex/tools/services.py +403 -334
  117. vortex/tools/storage.py +293 -358
  118. vortex/tools/surfex.py +24 -24
  119. vortex/tools/systems.py +1234 -643
  120. vortex/tools/targets.py +156 -100
  121. vortex/util/__init__.py +1 -1
  122. vortex/util/config.py +378 -327
  123. vortex/util/empty.py +2 -2
  124. vortex/util/helpers.py +56 -24
  125. vortex/util/introspection.py +18 -12
  126. vortex/util/iosponge.py +8 -4
  127. vortex/util/roles.py +4 -6
  128. vortex/util/storefunctions.py +39 -13
  129. vortex/util/structs.py +3 -3
  130. vortex/util/worker.py +29 -17
  131. vortex_nwp-2.1.0.dist-info/METADATA +67 -0
  132. vortex_nwp-2.1.0.dist-info/RECORD +144 -0
  133. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/WHEEL +1 -1
  134. vortex/layout/appconf.py +0 -109
  135. vortex/layout/jobs.py +0 -1276
  136. vortex/layout/nodes.py +0 -1424
  137. vortex/layout/subjobs.py +0 -464
  138. vortex_nwp-2.0.0b1.dist-info/METADATA +0 -50
  139. vortex_nwp-2.0.0b1.dist-info/RECORD +0 -146
  140. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info/licenses}/LICENSE +0 -0
  141. {vortex_nwp-2.0.0b1.dist-info → vortex_nwp-2.1.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1 @@
1
+ # Empty init file
@@ -23,30 +23,25 @@ class ServerSyncTool(footprints.FootprintBase):
23
23
  """
24
24
 
25
25
  _abstract = True
26
- _collector = ('serversynctool',)
26
+ _collector = ("serversynctool",)
27
27
  _footprint = dict(
28
- info='Abstract Server Synchronisation Tool',
28
+ info="Abstract Server Synchronisation Tool",
29
29
  attr=dict(
30
- method=dict(
31
- ),
30
+ method=dict(),
32
31
  medium=dict(
33
32
  optional=True,
34
33
  ),
35
- raiseonexit=dict(
36
- type=bool,
37
- optional=True,
38
- default=True
39
- ),
34
+ raiseonexit=dict(type=bool, optional=True, default=True),
40
35
  checkinterval=dict(
41
36
  type=int,
42
37
  optional=True,
43
38
  default=10,
44
39
  ),
45
- )
40
+ ),
46
41
  )
47
42
 
48
43
  def __init__(self, *args, **kw):
49
- logger.debug('Server Synchronisation Tool init %s', self.__class__)
44
+ logger.debug("Server Synchronisation Tool init %s", self.__class__)
50
45
  self._check_callback = lambda: True
51
46
  super().__init__(*args, **kw)
52
47
 
@@ -80,19 +75,19 @@ class ServerSyncSimpleSocket(ServerSyncTool):
80
75
  """
81
76
 
82
77
  _footprint = dict(
83
- info='Server Synchronisation Tool that uses a Socket',
78
+ info="Server Synchronisation Tool that uses a Socket",
84
79
  attr=dict(
85
80
  method=dict(
86
- values=['simple_socket'],
81
+ values=["simple_socket"],
87
82
  ),
88
83
  medium=dict(
89
84
  optional=False,
90
85
  ),
91
86
  tplname=dict(
92
87
  optional=True,
93
- default='@servsync-simplesocket.tpl',
88
+ default="@servsync-simplesocket.tpl",
94
89
  ),
95
- )
90
+ ),
96
91
  )
97
92
 
98
93
  def __init__(self, *args, **kw):
@@ -102,7 +97,7 @@ class ServerSyncSimpleSocket(ServerSyncTool):
102
97
  try:
103
98
  self._socket.bind((socket.getfqdn(), 0))
104
99
  except OSError:
105
- self._socket.bind(('localhost', 0))
100
+ self._socket.bind(("localhost", 0))
106
101
  self._socket.settimeout(self.checkinterval)
107
102
  self._socket.listen(1)
108
103
  # Current connection
@@ -110,11 +105,13 @@ class ServerSyncSimpleSocket(ServerSyncTool):
110
105
  # Create the script that will be called by the server
111
106
  t = sessions.current()
112
107
  tpl = config.load_template(t, self.tplname)
113
- with open(self.medium, 'w') as fd:
114
- fd.write(tpl.substitute(
115
- python=sys.executable,
116
- address=self._socket.getsockname(),
117
- ))
108
+ with open(self.medium, "w") as fd:
109
+ fd.write(
110
+ tpl.substitute(
111
+ python=sys.executable,
112
+ address=self._socket.getsockname(),
113
+ )
114
+ )
118
115
  t.sh.chmod(self.medium, 0o555)
119
116
 
120
117
  def __del__(self):
@@ -130,13 +127,13 @@ class ServerSyncSimpleSocket(ServerSyncTool):
130
127
  if self._socket_conn is not None:
131
128
  logger.info('Sending "%s" to the server.', mess)
132
129
  # NB: For send/recv, the settimeout also applies...
133
- self._socket_conn.send(mess.encode(encoding='utf-8'))
134
- repl = self._socket_conn.recv(255).decode(encoding='utf-8')
130
+ self._socket_conn.send(mess.encode(encoding="utf-8"))
131
+ repl = self._socket_conn.recv(255).decode(encoding="utf-8")
135
132
  logger.info('Server replied "%s" to %s.', repl, mess)
136
133
  self._socket_conn.close()
137
134
  self._socket_conn = None
138
- if repl != 'OK':
139
- raise ValueError(mess + ' failed')
135
+ if repl != "OK":
136
+ raise ValueError(mess + " failed")
140
137
  return True
141
138
  else:
142
139
  # This should not happen ! If we are sitting here, it's most likely
@@ -144,27 +141,31 @@ class ServerSyncSimpleSocket(ServerSyncTool):
144
141
  return False
145
142
 
146
143
  def trigger_wait(self):
147
- logger.info('Waiting for the server to complete')
144
+ logger.info("Waiting for the server to complete")
148
145
  while self._socket_conn is None and self._check_callback():
149
146
  try:
150
- self._socket_conn, addr = self._socket.accept() # @UnusedVariable
147
+ self._socket_conn, addr = (
148
+ self._socket.accept()
149
+ ) # @UnusedVariable
151
150
  except socket.timeout:
152
- logger.debug('Socket accept timed-out: checking for the server...')
151
+ logger.debug(
152
+ "Socket accept timed-out: checking for the server..."
153
+ )
153
154
  self._socket_conn = None
154
155
  if self._socket_conn is None:
155
156
  if self.raiseonexit:
156
- raise OSError('Apparently the server died.')
157
+ raise OSError("Apparently the server died.")
157
158
  else:
158
- logger.info('The server stopped.')
159
+ logger.info("The server stopped.")
159
160
  else:
160
161
  self._socket_conn.settimeout(self.checkinterval)
161
- logger.info('The server is now waiting')
162
+ logger.info("The server is now waiting")
162
163
 
163
164
  def trigger_run(self):
164
165
  # Tell the server that everything is ready
165
- self._command('STEP')
166
+ self._command("STEP")
166
167
  # Wait for the server to complete its work
167
168
  self.trigger_wait()
168
169
 
169
170
  def trigger_stop(self):
170
- return self._command('STOP')
171
+ return self._command("STOP")
vortex/config.py CHANGED
@@ -2,6 +2,7 @@
2
2
  the value of configuration options, respectively.
3
3
 
4
4
  """
5
+
5
6
  import tomli
6
7
 
7
8
  from bronx.fancies import loggers
@@ -19,6 +20,10 @@ VORTEX_CONFIG = {}
19
20
  logger = loggers.getLogger(__name__)
20
21
 
21
22
 
23
+ class ConfigurationError(Exception):
24
+ """Something is wrong with the provided configuration"""
25
+
26
+
22
27
  def load_config(configpath="vortex.toml"):
23
28
  """Load configuration from a TOML configuration file
24
29
 
@@ -38,21 +43,15 @@ def load_config(configpath="vortex.toml"):
38
43
  global VORTEX_CONFIG
39
44
  try:
40
45
  with open(configpath, "rb") as f:
41
- VORTEX_CONFIG = tomli.load(f)
46
+ VORTEX_CONFIG = tomli.load(f)
42
47
  print(f"Successfully read configuration file {configpath}")
43
48
  except FileNotFoundError:
44
- print(
45
- f"Could not read configuration file {configpath}"
46
- " (not found)."
47
- )
48
- print(
49
- "Use load_config(/path/to/config) to update the configuration"
50
- )
49
+ print(f"Could not read configuration file {configpath} (not found).")
50
+ print("Use load_config(/path/to/config) to update the configuration")
51
51
 
52
52
 
53
53
  def print_config():
54
- """Print configuration (key, value) pairs
55
- """
54
+ """Print configuration (key, value) pairs"""
56
55
  if VORTEX_CONFIG:
57
56
  for k, v in VORTEX_CONFIG:
58
57
  print(k.upper(), v)
@@ -66,18 +65,19 @@ def from_config(section, key=None):
66
65
  """
67
66
  try:
68
67
  subconfig = VORTEX_CONFIG[section]
69
- except KeyError as e:
70
- print(f"Could not find section {section} in configuration")
71
- raise(e)
72
-
68
+ except KeyError:
69
+ raise ConfigurationError(
70
+ f"Missing configuration section {section}",
71
+ )
73
72
  if not key:
74
73
  return subconfig
75
74
 
76
75
  try:
77
76
  value = subconfig[key]
78
- except KeyError as e :
79
- print(f"Could not find key {key} in section {section} of configuration")
80
- raise(e)
77
+ except KeyError:
78
+ raise ConfigurationError(
79
+ f"Missing configuration key {key} in section {section}",
80
+ )
81
81
  return value
82
82
 
83
83
 
@@ -87,9 +87,7 @@ def set_config(section, key, value):
87
87
  if section not in VORTEX_CONFIG.keys():
88
88
  VORTEX_CONFIG[section] = {}
89
89
  if key in VORTEX_CONFIG[section]:
90
- logger.warning(
91
- f"Updating existing configuration {section}:{key}"
92
- )
90
+ logger.warning(f"Updating existing configuration {section}:{key}")
93
91
  VORTEX_CONFIG[section][key] = value
94
92
 
95
93
 
@@ -108,8 +106,7 @@ def is_defined(section, key=None):
108
106
 
109
107
 
110
108
  def get_from_config_w_default(section, key, default):
111
- logger.info(f"Reading config value {section}.{key}")
112
109
  try:
113
110
  return from_config(section, key)
114
- except KeyError:
111
+ except ConfigurationError:
115
112
  return default
vortex/data/__init__.py CHANGED
@@ -4,10 +4,16 @@ Abstract classes involved in data management within VORTEX.
4
4
  Actual resources and custom providers should be defined in dedicated packages.
5
5
  """
6
6
 
7
- from . import handlers, resources, containers, contents, providers, \
8
- executables, stores, geometries
7
+ from . import handlers as handlers
8
+ from . import resources as resources
9
+ from . import containers as containers
10
+ from . import contents as contents
11
+ from . import providers as providers
12
+ from . import executables as executables
13
+ from . import stores as stores
14
+ from . import geometries as geometries
9
15
 
10
16
  #: No automatic export
11
17
  __all__ = []
12
18
 
13
- __tocinfoline__ = 'Abstract classes involved in data management within VORTEX'
19
+ __tocinfoline__ = "Abstract classes involved in data management within VORTEX"