hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  5. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  6. hpcflow/sdk/__init__.py +21 -15
  7. hpcflow/sdk/app.py +2133 -770
  8. hpcflow/sdk/cli.py +281 -250
  9. hpcflow/sdk/cli_common.py +6 -2
  10. hpcflow/sdk/config/__init__.py +1 -1
  11. hpcflow/sdk/config/callbacks.py +77 -42
  12. hpcflow/sdk/config/cli.py +126 -103
  13. hpcflow/sdk/config/config.py +578 -311
  14. hpcflow/sdk/config/config_file.py +131 -95
  15. hpcflow/sdk/config/errors.py +112 -85
  16. hpcflow/sdk/config/types.py +145 -0
  17. hpcflow/sdk/core/actions.py +1054 -994
  18. hpcflow/sdk/core/app_aware.py +24 -0
  19. hpcflow/sdk/core/cache.py +81 -63
  20. hpcflow/sdk/core/command_files.py +275 -185
  21. hpcflow/sdk/core/commands.py +111 -107
  22. hpcflow/sdk/core/element.py +724 -503
  23. hpcflow/sdk/core/enums.py +192 -0
  24. hpcflow/sdk/core/environment.py +74 -93
  25. hpcflow/sdk/core/errors.py +398 -51
  26. hpcflow/sdk/core/json_like.py +540 -272
  27. hpcflow/sdk/core/loop.py +380 -334
  28. hpcflow/sdk/core/loop_cache.py +160 -43
  29. hpcflow/sdk/core/object_list.py +370 -207
  30. hpcflow/sdk/core/parameters.py +728 -600
  31. hpcflow/sdk/core/rule.py +59 -41
  32. hpcflow/sdk/core/run_dir_files.py +33 -22
  33. hpcflow/sdk/core/task.py +1546 -1325
  34. hpcflow/sdk/core/task_schema.py +240 -196
  35. hpcflow/sdk/core/test_utils.py +126 -88
  36. hpcflow/sdk/core/types.py +387 -0
  37. hpcflow/sdk/core/utils.py +410 -305
  38. hpcflow/sdk/core/validation.py +82 -9
  39. hpcflow/sdk/core/workflow.py +1192 -1028
  40. hpcflow/sdk/core/zarr_io.py +98 -137
  41. hpcflow/sdk/demo/cli.py +46 -33
  42. hpcflow/sdk/helper/cli.py +18 -16
  43. hpcflow/sdk/helper/helper.py +75 -63
  44. hpcflow/sdk/helper/watcher.py +61 -28
  45. hpcflow/sdk/log.py +83 -59
  46. hpcflow/sdk/persistence/__init__.py +8 -31
  47. hpcflow/sdk/persistence/base.py +988 -586
  48. hpcflow/sdk/persistence/defaults.py +6 -0
  49. hpcflow/sdk/persistence/discovery.py +38 -0
  50. hpcflow/sdk/persistence/json.py +408 -153
  51. hpcflow/sdk/persistence/pending.py +158 -123
  52. hpcflow/sdk/persistence/store_resource.py +37 -22
  53. hpcflow/sdk/persistence/types.py +307 -0
  54. hpcflow/sdk/persistence/utils.py +14 -11
  55. hpcflow/sdk/persistence/zarr.py +477 -420
  56. hpcflow/sdk/runtime.py +44 -41
  57. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  58. hpcflow/sdk/submission/jobscript.py +444 -404
  59. hpcflow/sdk/submission/schedulers/__init__.py +133 -40
  60. hpcflow/sdk/submission/schedulers/direct.py +97 -71
  61. hpcflow/sdk/submission/schedulers/sge.py +132 -126
  62. hpcflow/sdk/submission/schedulers/slurm.py +263 -268
  63. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  64. hpcflow/sdk/submission/shells/__init__.py +14 -15
  65. hpcflow/sdk/submission/shells/base.py +102 -29
  66. hpcflow/sdk/submission/shells/bash.py +72 -55
  67. hpcflow/sdk/submission/shells/os_version.py +31 -30
  68. hpcflow/sdk/submission/shells/powershell.py +37 -29
  69. hpcflow/sdk/submission/submission.py +203 -257
  70. hpcflow/sdk/submission/types.py +143 -0
  71. hpcflow/sdk/typing.py +163 -12
  72. hpcflow/tests/conftest.py +8 -6
  73. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  74. hpcflow/tests/scripts/test_main_scripts.py +60 -30
  75. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
  76. hpcflow/tests/unit/test_action.py +86 -75
  77. hpcflow/tests/unit/test_action_rule.py +9 -4
  78. hpcflow/tests/unit/test_app.py +13 -6
  79. hpcflow/tests/unit/test_cli.py +1 -1
  80. hpcflow/tests/unit/test_command.py +71 -54
  81. hpcflow/tests/unit/test_config.py +20 -15
  82. hpcflow/tests/unit/test_config_file.py +21 -18
  83. hpcflow/tests/unit/test_element.py +58 -62
  84. hpcflow/tests/unit/test_element_iteration.py +3 -1
  85. hpcflow/tests/unit/test_element_set.py +29 -19
  86. hpcflow/tests/unit/test_group.py +4 -2
  87. hpcflow/tests/unit/test_input_source.py +116 -93
  88. hpcflow/tests/unit/test_input_value.py +29 -24
  89. hpcflow/tests/unit/test_json_like.py +44 -35
  90. hpcflow/tests/unit/test_loop.py +65 -58
  91. hpcflow/tests/unit/test_object_list.py +17 -12
  92. hpcflow/tests/unit/test_parameter.py +16 -7
  93. hpcflow/tests/unit/test_persistence.py +48 -35
  94. hpcflow/tests/unit/test_resources.py +20 -18
  95. hpcflow/tests/unit/test_run.py +8 -3
  96. hpcflow/tests/unit/test_runtime.py +2 -1
  97. hpcflow/tests/unit/test_schema_input.py +23 -15
  98. hpcflow/tests/unit/test_shell.py +3 -2
  99. hpcflow/tests/unit/test_slurm.py +8 -7
  100. hpcflow/tests/unit/test_submission.py +39 -19
  101. hpcflow/tests/unit/test_task.py +352 -247
  102. hpcflow/tests/unit/test_task_schema.py +33 -20
  103. hpcflow/tests/unit/test_utils.py +9 -11
  104. hpcflow/tests/unit/test_value_sequence.py +15 -12
  105. hpcflow/tests/unit/test_workflow.py +114 -83
  106. hpcflow/tests/unit/test_workflow_template.py +0 -1
  107. hpcflow/tests/workflows/test_jobscript.py +2 -1
  108. hpcflow/tests/workflows/test_workflows.py +18 -13
  109. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
  110. hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
  111. hpcflow/sdk/core/parallel.py +0 -21
  112. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  113. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
  114. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
  115. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/config/cli.py CHANGED
@@ -1,66 +1,70 @@
1
1
  """Module defining a function that returns the click CLI group for manipulating the app
2
2
  configuration."""
3
3
 
4
+ from __future__ import annotations
4
5
  import json
5
6
  import logging
6
7
  import warnings
7
8
  from functools import wraps
8
9
  from contextlib import contextmanager
10
+ from typing import TYPE_CHECKING
9
11
 
10
12
  import click
11
13
  from colorama import init as colorama_init
12
- from termcolor import colored
14
+ from termcolor import colored # type: ignore
13
15
 
14
- from hpcflow.sdk.core import utils
16
+ from hpcflow.sdk.core.utils import open_file
15
17
 
16
- from .errors import ConfigError
18
+ from hpcflow.sdk.config.errors import ConfigError
19
+ from hpcflow.sdk.config.config import Config
20
+
21
+ if TYPE_CHECKING:
22
+ from collections.abc import Callable, Sequence
23
+ from ..app import BaseApp
17
24
 
18
25
  logger = logging.getLogger(__name__)
19
26
 
20
27
  colorama_init(autoreset=True)
21
28
 
22
29
 
23
- def custom_warning_formatter(message, category, filename, lineno, file=None, line=None):
24
- """Simple warning formatter that shows just the warning type and the message. We use
25
- this in the CLI, to avoid producing distracting output."""
26
- return f"{colored(category.__name__, 'yellow')}: {message}\n"
27
-
28
-
29
- @contextmanager
30
- def warning_formatter(func=custom_warning_formatter):
31
- """Context manager for modifying the warnings formatter.
32
-
33
- Parameters
34
- ----------
35
- func : function to set as the `warnings.formatwarning` function.
36
-
37
- """
38
- existing_func = warnings.formatwarning
39
- try:
40
- warnings.formatwarning = func
41
- yield
42
- finally:
43
- warnings.formatwarning = existing_func
44
-
45
-
46
30
  def CLI_exception_wrapper_gen(*exception_cls):
47
31
  """
48
32
  Decorator factory that enhances the wrapped function to display a nice message on
49
33
  success or failure.
50
34
  """
51
35
 
52
- def CLI_exception_wrapper(func):
36
+ @contextmanager
37
+ def warning_formatter():
38
+ """
39
+ Context manager to apply a simple warning formatter that shows just the warning
40
+ type and the message. We use this in the CLI to avoid producing distracting
41
+ output.
42
+ """
43
+
44
+ def custom_warning_formatter(
45
+ message, category, filename, lineno, file=None, line=None
46
+ ):
47
+ return f"{colored(category.__name__, 'yellow')}: {message}\n"
48
+
49
+ existing_func = warnings.formatwarning
50
+ try:
51
+ warnings.formatwarning = custom_warning_formatter
52
+ yield
53
+ finally:
54
+ warnings.formatwarning = existing_func
55
+
56
+ def CLI_exception_wrapper(func: Callable):
53
57
  """Decorator
54
58
 
55
59
  Parameters
56
60
  ----------
57
61
  func
58
- Function that return a truthy value if the ???
62
+ Function that return a non-None value if the operation succeeds
59
63
  """
60
64
 
61
65
  @wraps(func)
62
66
  @click.pass_context
63
- def wrapper(ctx, *args, **kwargs):
67
+ def wrapper(ctx: click.Context, *args, **kwargs):
64
68
  try:
65
69
  with warning_formatter():
66
70
  out = func(*args, **kwargs)
@@ -76,25 +80,32 @@ def CLI_exception_wrapper_gen(*exception_cls):
76
80
  return CLI_exception_wrapper
77
81
 
78
82
 
79
- def get_config_CLI(app):
83
+ def get_config_CLI(app: BaseApp) -> click.Group:
80
84
  """Generate the configuration CLI for the app."""
81
85
 
82
- def show_all_config(ctx, param, value):
86
+ pass_config = click.make_pass_decorator(Config)
87
+
88
+ def find_config(ctx: click.Context) -> Config:
89
+ if (cfg := ctx.find_object(Config)) is None:
90
+ raise RuntimeError("no configuration defined")
91
+ return cfg
92
+
93
+ def show_all_config(ctx: click.Context, param, value: bool):
83
94
  if not value or ctx.resilient_parsing:
84
95
  return
85
- ctx.obj["config"]._show(config=True, metadata=False)
96
+ find_config(ctx)._show(config=True, metadata=False)
86
97
  ctx.exit()
87
98
 
88
- def show_all_metadata(ctx, param, value):
99
+ def show_all_metadata(ctx: click.Context, param, value: bool):
89
100
  if not value or ctx.resilient_parsing:
90
101
  return
91
- ctx.obj["config"]._show(config=False, metadata=True)
102
+ find_config(ctx)._show(config=False, metadata=True)
92
103
  ctx.exit()
93
104
 
94
- def show_config_file(ctx, param, value):
105
+ def show_config_file(ctx: click.Context, param, value: bool):
95
106
  if not value or ctx.resilient_parsing:
96
107
  return
97
- print(ctx.obj["config"].config_file_contents)
108
+ print(find_config(ctx).config_file_contents)
98
109
  ctx.exit()
99
110
 
100
111
  @click.group()
@@ -104,19 +115,18 @@ def get_config_CLI(app):
104
115
  help="Exclude a named get/set callback function during execution of the command.",
105
116
  )
106
117
  @click.pass_context
107
- def config(ctx, no_callback):
118
+ def config(ctx: click.Context, no_callback: Sequence[str]):
108
119
  """Configuration sub-command for getting and setting data in the configuration
109
120
  file(s)."""
110
- ctx.ensure_object(dict)
111
- ctx.obj["config"] = app.config
121
+ ctx.obj = app.config
112
122
  if no_callback:
113
- ctx.obj["config"]._disable_callbacks(no_callback)
123
+ app.config._disable_callbacks(no_callback)
114
124
 
115
125
  @config.command("list")
116
- @click.pass_context
117
- def config_list(ctx):
126
+ @pass_config
127
+ def config_list(config: Config):
118
128
  """Show a list of all configurable keys."""
119
- click.echo("\n".join(ctx.obj["config"].get_configurable()))
129
+ click.echo("\n".join(config.get_configurable()))
120
130
 
121
131
  @config.command("import")
122
132
  @click.argument("file_path")
@@ -139,10 +149,10 @@ def get_config_CLI(app):
139
149
  "config. If False, modify the currently loaded config."
140
150
  ),
141
151
  )
142
- @click.pass_context
143
- def import_from_file(ctx, file_path, rename, new):
152
+ @pass_config
153
+ def import_from_file(config: Config, file_path: str, rename: bool, new: bool):
144
154
  """Update the config file with keys from a YAML file."""
145
- ctx.obj["config"].import_from_file(file_path, rename=rename, make_new=new)
155
+ config.import_from_file(file_path, rename=rename, make_new=new)
146
156
 
147
157
  @config.command()
148
158
  @click.argument("name")
@@ -170,11 +180,11 @@ def get_config_CLI(app):
170
180
  help="Show the contents of the configuration file.",
171
181
  callback=show_config_file,
172
182
  )
173
- @click.pass_context
183
+ @pass_config
174
184
  @CLI_exception_wrapper_gen(ConfigError)
175
- def get(ctx, name):
185
+ def get(config: Config, name: str):
176
186
  """Show the value of the specified configuration item."""
177
- val = ctx.obj["config"].get(name)
187
+ val = config.get(name)
178
188
  if isinstance(val, list):
179
189
  val = "\n".join(str(i) for i in val)
180
190
  click.echo(val)
@@ -189,21 +199,24 @@ def get_config_CLI(app):
189
199
  default=False,
190
200
  help="Interpret VALUE as a JSON string.",
191
201
  )
192
- @click.pass_context
202
+ @pass_config
193
203
  @CLI_exception_wrapper_gen(ConfigError)
194
- def set(ctx, name, value, is_json):
204
+ def set(config: Config, name: str, value: str, is_json: bool):
195
205
  """Set and save the value of the specified configuration item."""
196
- ctx.obj["config"].set(name, value, is_json)
197
- ctx.obj["config"].save()
206
+ if is_json:
207
+ config.set(name, value, is_json=True)
208
+ else:
209
+ config.set(name, value, is_json=False)
210
+ config.save()
198
211
 
199
212
  @config.command()
200
213
  @click.argument("name")
201
- @click.pass_context
214
+ @pass_config
202
215
  @CLI_exception_wrapper_gen(ConfigError)
203
- def unset(ctx, name):
216
+ def unset(config: Config, name: str):
204
217
  """Unset and save the value of the specified configuration item."""
205
- ctx.obj["config"].unset(name)
206
- ctx.obj["config"].save()
218
+ config.unset(name)
219
+ config.save()
207
220
 
208
221
  @config.command()
209
222
  @click.argument("name")
@@ -215,16 +228,19 @@ def get_config_CLI(app):
215
228
  default=False,
216
229
  help="Interpret VALUE as a JSON string.",
217
230
  )
218
- @click.pass_context
231
+ @pass_config
219
232
  @CLI_exception_wrapper_gen(ConfigError)
220
- def append(ctx, name, value, is_json):
233
+ def append(config: Config, name: str, value: str, is_json: bool):
221
234
  """Append a new value to the specified configuration item.
222
235
 
223
236
  NAME is the dot-delimited path to the list to be appended to.
224
237
 
225
238
  """
226
- ctx.obj["config"].append(name, value, is_json)
227
- ctx.obj["config"].save()
239
+ if is_json:
240
+ config.append(name, value, is_json=True)
241
+ else:
242
+ config.append(name, value, is_json=False)
243
+ config.save()
228
244
 
229
245
  @config.command()
230
246
  @click.argument("name")
@@ -236,30 +252,33 @@ def get_config_CLI(app):
236
252
  default=False,
237
253
  help="Interpret VALUE as a JSON string.",
238
254
  )
239
- @click.pass_context
255
+ @pass_config
240
256
  @CLI_exception_wrapper_gen(ConfigError)
241
- def prepend(ctx, name, value, is_json):
257
+ def prepend(config: Config, name: str, value: str, is_json: bool):
242
258
  """Prepend a new value to the specified configuration item.
243
259
 
244
260
  NAME is the dot-delimited path to the list to be prepended to.
245
261
 
246
262
  """
247
- ctx.obj["config"].prepend(name, value, is_json)
248
- ctx.obj["config"].save()
263
+ if is_json:
264
+ config.prepend(name, value, is_json=True)
265
+ else:
266
+ config.prepend(name, value, is_json=False)
267
+ config.save()
249
268
 
250
269
  @config.command(context_settings={"ignore_unknown_options": True})
251
270
  @click.argument("name")
252
271
  @click.argument("index", type=click.types.INT)
253
- @click.pass_context
272
+ @pass_config
254
273
  @CLI_exception_wrapper_gen(ConfigError)
255
- def pop(ctx, name, index):
274
+ def pop(config: Config, name: str, index: int):
256
275
  """Remove a value from a list-like configuration item.
257
276
 
258
277
  NAME is the dot-delimited path to the list to be modified.
259
278
 
260
279
  """
261
- ctx.obj["config"].pop(name, index)
262
- ctx.obj["config"].save()
280
+ config.pop(name, index)
281
+ config.save()
263
282
 
264
283
  @config.command()
265
284
  @click.argument("name")
@@ -271,80 +290,84 @@ def get_config_CLI(app):
271
290
  default=False,
272
291
  help="Interpret VALUE as a JSON string.",
273
292
  )
274
- @click.pass_context
293
+ @pass_config
275
294
  @CLI_exception_wrapper_gen(ConfigError)
276
- def update(ctx, name, value, is_json):
295
+ def update(config: Config, name: str, value: str, is_json: bool):
277
296
  """Update a map-like value in the configuration.
278
297
 
279
298
  NAME is the dot-delimited path to the map to be updated.
280
299
 
281
300
  """
282
- ctx.obj["config"].update(name, value, is_json)
283
- ctx.obj["config"].save()
301
+ if is_json:
302
+ config.update(name, value, is_json=True)
303
+ else:
304
+ config.update(name, value, is_json=False)
305
+ config.save()
284
306
 
285
307
  @config.command()
286
308
  @click.argument("name")
287
309
  @click.option("--defaults")
288
- @click.pass_context
310
+ @pass_config
289
311
  @CLI_exception_wrapper_gen(ConfigError)
290
- def add_scheduler(ctx, name, defaults):
312
+ def add_scheduler(config: Config, name: str, defaults: str | None):
291
313
  if defaults:
292
- defaults = json.loads(defaults)
314
+ loaded_defaults: dict = json.loads(defaults)
293
315
  else:
294
- defaults = {}
295
- ctx.obj["config"].add_scheduler(name, **defaults)
296
- ctx.obj["config"].save()
316
+ loaded_defaults = {}
317
+ config.add_scheduler(name, **loaded_defaults)
318
+ config.save()
297
319
 
298
320
  @config.command()
299
321
  @click.argument("name")
300
322
  @click.option("--defaults")
301
- @click.pass_context
323
+ @pass_config
302
324
  @CLI_exception_wrapper_gen(ConfigError)
303
- def add_shell(ctx, name, defaults):
325
+ def add_shell(config: Config, name: str, defaults: str | None):
304
326
  if defaults:
305
- defaults = json.loads(defaults)
327
+ loaded_defaults: dict = json.loads(defaults)
306
328
  else:
307
- defaults = {}
308
- ctx.obj["config"].add_shell(name, **defaults)
309
- ctx.obj["config"].save()
329
+ loaded_defaults = {}
330
+ config.add_shell(name, **loaded_defaults)
331
+ config.save()
310
332
 
311
333
  @config.command()
312
334
  @click.option("--defaults")
313
- @click.pass_context
335
+ @pass_config
314
336
  @CLI_exception_wrapper_gen(ConfigError)
315
- def add_shell_wsl(ctx, defaults):
337
+ def add_shell_wsl(config: Config, defaults: str | None):
316
338
  if defaults:
317
- defaults = json.loads(defaults)
339
+ loaded_defaults: dict = json.loads(defaults)
318
340
  else:
319
- defaults = {}
320
- ctx.obj["config"].add_shell_WSL(**defaults)
321
- ctx.obj["config"].save()
341
+ loaded_defaults = {}
342
+ config.add_shell_WSL(**loaded_defaults)
343
+ config.save()
322
344
 
323
345
  @config.command()
324
346
  @click.argument("sha")
325
- @click.pass_context
347
+ @pass_config
326
348
  @CLI_exception_wrapper_gen(ConfigError)
327
- def set_github_demo_data_dir(ctx, sha):
328
- ctx.obj["config"].set_github_demo_data_dir(sha=sha)
329
- ctx.obj["config"].save()
349
+ def set_github_demo_data_dir(config: Config, sha: str):
350
+ config.set_github_demo_data_dir(sha=sha)
351
+ config.save()
330
352
 
331
353
  @config.command()
332
354
  def load_data_files():
333
355
  """Check we can load the data files (e.g. task schema files) as specified in the
334
356
  configuration."""
335
357
  app.load_data_files()
358
+ # FIXME: No such method?
336
359
 
337
360
  @config.command()
338
361
  @click.option("--path", is_flag=True, default=False)
339
- @click.pass_context
340
- def open(ctx, path=False):
362
+ @pass_config
363
+ def open(config: Config, path: bool = False):
341
364
  """Alias for `{package_name} open config`: open the configuration file, or retrieve
342
365
  it's path."""
343
- file_path = ctx.obj["config"].get("config_file_path")
366
+ file_path = config.get("config_file_path")
344
367
  if path:
345
368
  click.echo(file_path)
346
369
  else:
347
- utils.open_file(file_path)
370
+ open_file(file_path)
348
371
 
349
372
  @config.command()
350
373
  @click.argument("known_name")
@@ -356,10 +379,10 @@ def get_config_CLI(app):
356
379
  "files."
357
380
  ),
358
381
  )
359
- @click.pass_context
360
- def init(ctx, known_name, path):
361
- ctx.obj["config"].init(known_name=known_name, path=path)
382
+ @pass_config
383
+ def init(config: Config, known_name: str, path: str | None):
384
+ config.init(known_name=known_name, path=path)
362
385
 
363
- open.help = open.help.format(package_name=app.package_name)
386
+ open.help = (open.help or "").format(package_name=app.package_name)
364
387
 
365
388
  return config