hpcflow-new2 0.2.0a178__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
  67. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/rule.py CHANGED
@@ -1,3 +1,7 @@
1
+ """
2
+ Rules apply conditions to workflow elements or loops.
3
+ """
4
+
1
5
  from __future__ import annotations
2
6
  from typing import Dict, Optional, Union
3
7
 
@@ -11,7 +15,27 @@ from hpcflow.sdk.log import TimeIt
11
15
 
12
16
 
13
17
  class Rule(JSONLike):
14
- """Class to represent a testable condition on an element iteration or run."""
18
+ """
19
+ Class to represent a testable condition on an element iteration or run.
20
+
21
+ Exactly one of ``check_exists``, ``check_missing`` and ``condition`` must be provided.
22
+
23
+ Parameters
24
+ ----------
25
+ check_exists: str
26
+ If set, check this attribute exists.
27
+ check_missing: str
28
+ If set, check this attribute does *not* exist.
29
+ path: str
30
+ Where to look up the attribute to check.
31
+ If not specified, determined by context.
32
+ condition: ConditionLike
33
+ A general condition to check (or kwargs used to generate one).
34
+ cast: str
35
+ If set, a cast to apply prior to running the general check.
36
+ doc: str
37
+ Optional descriptive text.
38
+ """
15
39
 
16
40
  def __init__(
17
41
  self,
@@ -31,11 +55,17 @@ class Rule(JSONLike):
31
55
  if isinstance(condition, dict):
32
56
  condition = ConditionLike.from_json_like(condition)
33
57
 
58
+ #: If set, this rule checks this attribute exists.
34
59
  self.check_exists = check_exists
60
+ #: If set, this rule checks this attribute does *not* exist.
35
61
  self.check_missing = check_missing
62
+ #: Where to look up the attribute to check (if not determined by context).
36
63
  self.path = path
64
+ #: A general condition for this rule to check.
37
65
  self.condition = condition
66
+ #: If set, a cast to apply prior to running the general check.
38
67
  self.cast = cast
68
+ #: Optional descriptive text.
39
69
  self.doc = doc
40
70
 
41
71
  def __repr__(self):
@@ -1,3 +1,7 @@
1
+ """
2
+ Model of files in the run directory.
3
+ """
4
+
1
5
  import re
2
6
  from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
3
7
 
@@ -8,7 +12,7 @@ class RunDirAppFiles:
8
12
 
9
13
  _app_attr = "app"
10
14
 
11
- CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*"
15
+ _CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*"
12
16
 
13
17
  @classmethod
14
18
  def get_log_file_name(cls):
@@ -22,10 +26,16 @@ class RunDirAppFiles:
22
26
 
23
27
  @staticmethod
24
28
  def get_run_file_prefix(js_idx: int, js_action_idx: int):
29
+ """
30
+ Get the common prefix for files associated with a run.
31
+ """
25
32
  return f"js_{js_idx}_act_{js_action_idx}"
26
33
 
27
34
  @classmethod
28
35
  def get_commands_file_name(cls, js_idx: int, js_action_idx: int, shell):
36
+ """
37
+ Get the name of the file containing commands.
38
+ """
29
39
  return cls.get_run_file_prefix(js_idx, js_action_idx) + shell.JS_EXT
30
40
 
31
41
  @classmethod
@@ -56,7 +66,7 @@ class RunDirAppFiles:
56
66
  if (
57
67
  k == cls.get_log_file_name()
58
68
  or k == cls.get_std_file_name()
59
- or re.match(cls.CMD_FILES_RE_PATTERN, k)
69
+ or re.match(cls._CMD_FILES_RE_PATTERN, k)
60
70
  ):
61
71
  ss_js["data"].pop(k)
62
72