hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
  67. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,7 @@
1
+ """
2
+ Configuration file adapter.
3
+ """
4
+
1
5
  from __future__ import annotations
2
6
 
3
7
  import copy
@@ -26,10 +30,23 @@ from .errors import (
26
30
 
27
31
 
28
32
  class ConfigFile:
29
- """Configuration file."""
33
+ """
34
+ Configuration file.
35
+
36
+ Parameters
37
+ ----------
38
+ directory:
39
+ The directory containing the configuration file.
40
+ logger:
41
+ Where to log messages.
42
+ config_options:
43
+ Configuration options.
44
+ """
30
45
 
31
46
  def __init__(self, directory, logger, config_options):
47
+ #: Where to log messages.
32
48
  self.logger = logger
49
+ #: The directory containing the configuration file.
33
50
  self.directory = self._resolve_config_dir(
34
51
  config_opt=config_options,
35
52
  logger=self.logger,
@@ -39,9 +56,13 @@ class ConfigFile:
39
56
  self._configs = []
40
57
 
41
58
  # set by _load_file_data:
59
+ #: The path to the config file.
42
60
  self.path = None
61
+ #: The cached contents of the config file.
43
62
  self.contents = None
63
+ #: The parsed contents of the config file.
44
64
  self.data = None
65
+ #: The parsed contents of the config file where the alternate parser was used.
45
66
  self.data_rt = None
46
67
 
47
68
  self._load_file_data(config_options)
@@ -105,12 +126,31 @@ class ConfigFile:
105
126
  return file_schema
106
127
 
107
128
  def get_invoc_data(self, config_key):
129
+ """
130
+ Get the invocation data for the given configuration.
131
+
132
+ Parameters
133
+ ----------
134
+ config_key: str
135
+ The name of the configuration within the configuration file.
136
+ """
108
137
  return self.data["configs"][config_key]
109
138
 
110
139
  def get_invocation(self, config_key):
140
+ """
141
+ Get the invocation for the given configuration.
142
+
143
+ Parameters
144
+ ----------
145
+ config_key: str
146
+ The name of the configuration within the configuration file.
147
+ """
111
148
  return self.get_invoc_data(config_key)["invocation"]
112
149
 
113
150
  def save(self):
151
+ """
152
+ Write the (modified) configuration to the configuration file.
153
+ """
114
154
  new_data = copy.deepcopy(self.data)
115
155
  new_data_rt = copy.deepcopy(self.data_rt)
116
156
  new_contents = ""
@@ -271,6 +311,9 @@ class ConfigFile:
271
311
 
272
312
  @staticmethod
273
313
  def get_config_file_path(directory):
314
+ """
315
+ Get the path to the configuration file.
316
+ """
274
317
  # Try both ".yml" and ".yaml" extensions:
275
318
  path_yaml = directory.joinpath("config.yaml")
276
319
  if path_yaml.is_file():
@@ -307,11 +350,36 @@ class ConfigFile:
307
350
  def get_config_item(
308
351
  self, config_key, name, raise_on_missing=False, default_value=None
309
352
  ):
353
+ """
354
+ Get a configuration item.
355
+
356
+ Parameters
357
+ ----------
358
+ config_key: str
359
+ The name of the configuration within the configuration file.
360
+ name: str
361
+ The name of the configuration item.
362
+ raise_on_missing: bool
363
+ Whether to raise an error if the config item is absent.
364
+ default_value:
365
+ The default value to use when the config item is absent
366
+ (and ``raise_on_missing`` is not specified).
367
+ """
310
368
  if raise_on_missing and name not in self.get_invoc_data(config_key)["config"]:
311
369
  raise ValueError(f"missing from file: {name!r}")
312
370
  return self.get_invoc_data(config_key)["config"].get(name, default_value)
313
371
 
314
372
  def is_item_set(self, config_key, name):
373
+ """
374
+ Determine if a configuration item is set.
375
+
376
+ Parameters
377
+ ----------
378
+ config_key: str
379
+ The name of the configuration within the configuration file.
380
+ name: str
381
+ The name of the configuration item.
382
+ """
315
383
  try:
316
384
  self.get_config_item(config_key, name, raise_on_missing=True)
317
385
  except ValueError:
@@ -319,7 +387,16 @@ class ConfigFile:
319
387
  return True
320
388
 
321
389
  def rename_config_key(self, config_key: str, new_config_key: str):
322
- """Change the config key of the loaded config."""
390
+ """
391
+ Change the config key of the loaded config.
392
+
393
+ Parameters
394
+ ----------
395
+ config_key: str
396
+ The old name of the configuration within the configuration file.
397
+ new_config_key: str
398
+ The new name of the configuration.
399
+ """
323
400
 
324
401
  new_data = copy.deepcopy(self.data)
325
402
  new_data_rt = copy.deepcopy(self.data_rt)
@@ -342,7 +419,18 @@ class ConfigFile:
342
419
  environment_setup: Optional[str] = None,
343
420
  match: Optional[Dict] = None,
344
421
  ):
345
- """Modify the invocation parameters of the loaded config."""
422
+ """
423
+ Modify the invocation parameters of the loaded config.
424
+
425
+ Parameters
426
+ ----------
427
+ config_key: str
428
+ The name of the configuration within the configuration file.
429
+ environment_setup:
430
+ The new value of the ``environment_setup`` key.
431
+ match:
432
+ The new values to merge into the ``match`` key.
433
+ """
346
434
 
347
435
  new_data = copy.deepcopy(self.data)
348
436
  new_data_rt = copy.deepcopy(self.data_rt)
@@ -1,3 +1,8 @@
1
+ """
2
+ Miscellaneous configuration-related errors.
3
+ """
4
+
5
+
1
6
  class ConfigError(Exception):
2
7
  """Raised when a valid configuration can not be associated with the current
3
8
  invocation."""
@@ -6,6 +11,10 @@ class ConfigError(Exception):
6
11
 
7
12
 
8
13
  class ConfigUnknownItemError(ConfigError):
14
+ """
15
+ Raised when the configuration contains an unknown item.
16
+ """
17
+
9
18
  def __init__(self, name, message=None):
10
19
  self.message = message or (
11
20
  f"Specified name {name!r} is not a valid meta-data or configurable "
@@ -15,6 +24,10 @@ class ConfigUnknownItemError(ConfigError):
15
24
 
16
25
 
17
26
  class ConfigUnknownOverrideError(ConfigError):
27
+ """
28
+ Raised when the configuration override contains an unknown item.
29
+ """
30
+
18
31
  def __init__(self, name, message=None):
19
32
  self.message = message or (
20
33
  f"Specified configuration override {name!r} is not a valid configurable item."
@@ -23,22 +36,38 @@ class ConfigUnknownOverrideError(ConfigError):
23
36
 
24
37
 
25
38
  class ConfigNonConfigurableError(ConfigError):
39
+ """
40
+ Raised when the configuration contains an item that can't be configured.
41
+ """
42
+
26
43
  def __init__(self, name, message=None):
27
44
  self.message = message or (f"Specified name {name!r} is not a configurable item.")
28
45
  super().__init__(self.message)
29
46
 
30
47
 
31
48
  class ConfigItemAlreadyUnsetError(ConfigError):
49
+ """
50
+ Raised when the configuration tries to unset an unset item.
51
+ """
52
+
32
53
  def __init__(self, name, message=None):
33
54
  self.message = message or f"Configuration item {name!r} is already not set."
34
55
  super().__init__(self.message)
35
56
 
36
57
 
37
58
  class ConfigFileValidationError(ConfigError):
59
+ """
60
+ Raised when the configuration file fails validation.
61
+ """
62
+
38
63
  pass
39
64
 
40
65
 
41
66
  class ConfigItemCallbackError(ConfigError):
67
+ """
68
+ Raised when a configuration callback errors.
69
+ """
70
+
42
71
  def __init__(self, name, callback, err, message=None):
43
72
  self.message = message or (
44
73
  f"Callback function {callback.__name__!r} for configuration item {name!r} "
@@ -59,6 +88,10 @@ class ConfigFileInvocationIncompatibleError(ConfigError):
59
88
 
60
89
 
61
90
  class ConfigFileInvocationUnknownMatchKey(ConfigError):
91
+ """
92
+ Raised when the configuration contains an invalid match key.
93
+ """
94
+
62
95
  def __init__(self, match_key, message=None):
63
96
  self.message = message or (
64
97
  f"Specified match key ({match_key!r}) is not a valid run time info "
@@ -8,5 +8,7 @@ https://tldp.org/LDP/abs/html/exitcodes.html
8
8
 
9
9
  """
10
10
 
11
+ #: Formats supported for templates.
11
12
  ALL_TEMPLATE_FORMATS = ("yaml", "json")
13
+ #: The exit code used by an EAR when it aborts.
12
14
  ABORT_EXIT_CODE = 64