hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  5. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  6. hpcflow/sdk/__init__.py +21 -15
  7. hpcflow/sdk/app.py +2133 -770
  8. hpcflow/sdk/cli.py +281 -250
  9. hpcflow/sdk/cli_common.py +6 -2
  10. hpcflow/sdk/config/__init__.py +1 -1
  11. hpcflow/sdk/config/callbacks.py +77 -42
  12. hpcflow/sdk/config/cli.py +126 -103
  13. hpcflow/sdk/config/config.py +578 -311
  14. hpcflow/sdk/config/config_file.py +131 -95
  15. hpcflow/sdk/config/errors.py +112 -85
  16. hpcflow/sdk/config/types.py +145 -0
  17. hpcflow/sdk/core/actions.py +1054 -994
  18. hpcflow/sdk/core/app_aware.py +24 -0
  19. hpcflow/sdk/core/cache.py +81 -63
  20. hpcflow/sdk/core/command_files.py +275 -185
  21. hpcflow/sdk/core/commands.py +111 -107
  22. hpcflow/sdk/core/element.py +724 -503
  23. hpcflow/sdk/core/enums.py +192 -0
  24. hpcflow/sdk/core/environment.py +74 -93
  25. hpcflow/sdk/core/errors.py +398 -51
  26. hpcflow/sdk/core/json_like.py +540 -272
  27. hpcflow/sdk/core/loop.py +380 -334
  28. hpcflow/sdk/core/loop_cache.py +160 -43
  29. hpcflow/sdk/core/object_list.py +370 -207
  30. hpcflow/sdk/core/parameters.py +728 -600
  31. hpcflow/sdk/core/rule.py +59 -41
  32. hpcflow/sdk/core/run_dir_files.py +33 -22
  33. hpcflow/sdk/core/task.py +1546 -1325
  34. hpcflow/sdk/core/task_schema.py +240 -196
  35. hpcflow/sdk/core/test_utils.py +126 -88
  36. hpcflow/sdk/core/types.py +387 -0
  37. hpcflow/sdk/core/utils.py +410 -305
  38. hpcflow/sdk/core/validation.py +82 -9
  39. hpcflow/sdk/core/workflow.py +1192 -1028
  40. hpcflow/sdk/core/zarr_io.py +98 -137
  41. hpcflow/sdk/demo/cli.py +46 -33
  42. hpcflow/sdk/helper/cli.py +18 -16
  43. hpcflow/sdk/helper/helper.py +75 -63
  44. hpcflow/sdk/helper/watcher.py +61 -28
  45. hpcflow/sdk/log.py +83 -59
  46. hpcflow/sdk/persistence/__init__.py +8 -31
  47. hpcflow/sdk/persistence/base.py +988 -586
  48. hpcflow/sdk/persistence/defaults.py +6 -0
  49. hpcflow/sdk/persistence/discovery.py +38 -0
  50. hpcflow/sdk/persistence/json.py +408 -153
  51. hpcflow/sdk/persistence/pending.py +158 -123
  52. hpcflow/sdk/persistence/store_resource.py +37 -22
  53. hpcflow/sdk/persistence/types.py +307 -0
  54. hpcflow/sdk/persistence/utils.py +14 -11
  55. hpcflow/sdk/persistence/zarr.py +477 -420
  56. hpcflow/sdk/runtime.py +44 -41
  57. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  58. hpcflow/sdk/submission/jobscript.py +444 -404
  59. hpcflow/sdk/submission/schedulers/__init__.py +133 -40
  60. hpcflow/sdk/submission/schedulers/direct.py +97 -71
  61. hpcflow/sdk/submission/schedulers/sge.py +132 -126
  62. hpcflow/sdk/submission/schedulers/slurm.py +263 -268
  63. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  64. hpcflow/sdk/submission/shells/__init__.py +14 -15
  65. hpcflow/sdk/submission/shells/base.py +102 -29
  66. hpcflow/sdk/submission/shells/bash.py +72 -55
  67. hpcflow/sdk/submission/shells/os_version.py +31 -30
  68. hpcflow/sdk/submission/shells/powershell.py +37 -29
  69. hpcflow/sdk/submission/submission.py +203 -257
  70. hpcflow/sdk/submission/types.py +143 -0
  71. hpcflow/sdk/typing.py +163 -12
  72. hpcflow/tests/conftest.py +8 -6
  73. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  74. hpcflow/tests/scripts/test_main_scripts.py +60 -30
  75. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
  76. hpcflow/tests/unit/test_action.py +86 -75
  77. hpcflow/tests/unit/test_action_rule.py +9 -4
  78. hpcflow/tests/unit/test_app.py +13 -6
  79. hpcflow/tests/unit/test_cli.py +1 -1
  80. hpcflow/tests/unit/test_command.py +71 -54
  81. hpcflow/tests/unit/test_config.py +20 -15
  82. hpcflow/tests/unit/test_config_file.py +21 -18
  83. hpcflow/tests/unit/test_element.py +58 -62
  84. hpcflow/tests/unit/test_element_iteration.py +3 -1
  85. hpcflow/tests/unit/test_element_set.py +29 -19
  86. hpcflow/tests/unit/test_group.py +4 -2
  87. hpcflow/tests/unit/test_input_source.py +116 -93
  88. hpcflow/tests/unit/test_input_value.py +29 -24
  89. hpcflow/tests/unit/test_json_like.py +44 -35
  90. hpcflow/tests/unit/test_loop.py +65 -58
  91. hpcflow/tests/unit/test_object_list.py +17 -12
  92. hpcflow/tests/unit/test_parameter.py +16 -7
  93. hpcflow/tests/unit/test_persistence.py +48 -35
  94. hpcflow/tests/unit/test_resources.py +20 -18
  95. hpcflow/tests/unit/test_run.py +8 -3
  96. hpcflow/tests/unit/test_runtime.py +2 -1
  97. hpcflow/tests/unit/test_schema_input.py +23 -15
  98. hpcflow/tests/unit/test_shell.py +3 -2
  99. hpcflow/tests/unit/test_slurm.py +8 -7
  100. hpcflow/tests/unit/test_submission.py +39 -19
  101. hpcflow/tests/unit/test_task.py +352 -247
  102. hpcflow/tests/unit/test_task_schema.py +33 -20
  103. hpcflow/tests/unit/test_utils.py +9 -11
  104. hpcflow/tests/unit/test_value_sequence.py +15 -12
  105. hpcflow/tests/unit/test_workflow.py +114 -83
  106. hpcflow/tests/unit/test_workflow_template.py +0 -1
  107. hpcflow/tests/workflows/test_jobscript.py +2 -1
  108. hpcflow/tests/workflows/test_workflows.py +18 -13
  109. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
  110. hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
  111. hpcflow/sdk/core/parallel.py +0 -21
  112. hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
  113. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
  114. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
  115. {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -4,9 +4,8 @@ from hpcflow.sdk import sdk_classes
4
4
 
5
5
 
6
6
  # most of the modules in `sdk_classes` are imported on-demand via the app object:
7
- hiddenimports = list(sdk_classes.values())
8
-
9
- hiddenimports += [
7
+ hiddenimports = [
8
+ *sdk_classes.values(),
10
9
  "hpcflow.sdk.data",
11
10
  "hpcflow.data.demo_data_manifest",
12
11
  "hpcflow.data.scripts",
@@ -19,13 +18,16 @@ hiddenimports += [
19
18
  "fsspec.implementations.github", # for GitHub fsspec file system
20
19
  ]
21
20
 
22
- py_include_kwargs = dict(include_py_files=True, excludes=("**/__pycache__",))
23
21
  datas = (
24
22
  collect_data_files("hpcflow.sdk.data")
25
23
  + collect_data_files("hpcflow.data.demo_data_manifest")
26
- + collect_data_files("hpcflow.data.scripts", **py_include_kwargs)
24
+ + collect_data_files(
25
+ "hpcflow.data.scripts", include_py_files=True, excludes=("**/__pycache__",)
26
+ )
27
27
  + collect_data_files("hpcflow.data.template_components")
28
28
  + collect_data_files("hpcflow.data.workflows")
29
- + collect_data_files("hpcflow.tests", **py_include_kwargs)
29
+ + collect_data_files(
30
+ "hpcflow.tests", include_py_files=True, excludes=("**/__pycache__",)
31
+ )
30
32
  + collect_data_files("hpcflow.tests.data")
31
33
  )
hpcflow/_version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.2.0a188"
1
+ __version__ = "0.2.0a190"
hpcflow/app.py CHANGED
@@ -1,3 +1,4 @@
1
+ from __future__ import annotations
1
2
  from hpcflow import __version__, _app_name
2
3
  from hpcflow.sdk import app as sdk_app
3
4
  from hpcflow.sdk.config import ConfigOptions
@@ -1,4 +1,4 @@
1
- import h5py
1
+ import h5py # type: ignore[import-untyped]
2
2
 
3
3
 
4
4
  def main_script_test_hdf5_in_obj(_input_files):
@@ -1,4 +1,4 @@
1
- import h5py
1
+ import h5py # type: ignore[import-untyped]
2
2
 
3
3
 
4
4
  def main_script_test_hdf5_out_obj(p1, _output_files):
hpcflow/sdk/__init__.py CHANGED
@@ -1,15 +1,18 @@
1
1
  """Sub-package to define an extensible hpcflow application."""
2
2
 
3
+ from __future__ import annotations
4
+ from collections.abc import Mapping
3
5
  import logging
4
6
  import os
5
7
  import sys
8
+ from typing import Final
6
9
 
7
10
  #: Classes used in the construction of a workflow.
8
11
  #: :meta hide-value:
9
- sdk_classes = {
12
+ sdk_classes: Final[Mapping[str, str]] = {
10
13
  "Workflow": "hpcflow.sdk.core.workflow",
11
14
  "Task": "hpcflow.sdk.core.task",
12
- "ActionScopeType": "hpcflow.sdk.core.actions",
15
+ "ActionScopeType": "hpcflow.sdk.core.enums",
13
16
  "ActionScope": "hpcflow.sdk.core.actions",
14
17
  "ActionRule": "hpcflow.sdk.core.actions",
15
18
  "Rule": "hpcflow.sdk.core.rule",
@@ -68,9 +71,9 @@ sdk_classes = {
68
71
  "ValuePerturbation": "hpcflow.sdk.core.parameters",
69
72
  "InputValue": "hpcflow.sdk.core.parameters",
70
73
  "ResourceSpec": "hpcflow.sdk.core.parameters",
71
- "TaskSourceType": "hpcflow.sdk.core.parameters",
72
- "InputSourceType": "hpcflow.sdk.core.parameters",
73
- "ParameterPropagationMode": "hpcflow.sdk.core.parameters",
74
+ "TaskSourceType": "hpcflow.sdk.core.enums",
75
+ "InputSourceType": "hpcflow.sdk.core.enums",
76
+ "ParameterPropagationMode": "hpcflow.sdk.core.enums",
74
77
  "InputSource": "hpcflow.sdk.core.parameters",
75
78
  "TaskObjective": "hpcflow.sdk.core.task_schema",
76
79
  "TaskSchema": "hpcflow.sdk.core.task_schema",
@@ -87,6 +90,7 @@ sdk_classes = {
87
90
  "WorkflowBlueprint": "hpcflow.sdk.core.workflow",
88
91
  "Jobscript": "hpcflow.sdk.submission.jobscript",
89
92
  "Submission": "hpcflow.sdk.submission.submission",
93
+ "QueuedScheduler": "hpcflow.sdk.submission.schedulers",
90
94
  "DirectWindows": "hpcflow.sdk.submission.schedulers.direct",
91
95
  "DirectPosix": "hpcflow.sdk.submission.schedulers.direct",
92
96
  "SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
@@ -98,7 +102,7 @@ sdk_classes = {
98
102
  # these are defined as `BaseApp` methods with an underscore prefix:
99
103
  #: Functions exported by the application.
100
104
  #: :meta hide-value:
101
- sdk_funcs = (
105
+ sdk_funcs: Final[tuple[str, ...]] = (
102
106
  "make_workflow",
103
107
  "make_demo_workflow",
104
108
  "make_and_submit_workflow",
@@ -114,24 +118,26 @@ sdk_funcs = (
114
118
  "cancel",
115
119
  )
116
120
 
117
- _SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
118
121
 
119
-
120
- def get_SDK_logger(name=None):
122
+ def get_SDK_logger(name: str | None = None) -> logging.Logger:
121
123
  """Get a logger with prefix of "hpcflow_sdk" instead of "hpcflow.sdk" to ensure the
122
124
  handlers of the SDK logger and app logger are distinct."""
123
125
  name = ".".join(["hpcflow_sdk"] + (name or __name__).split(".")[2:])
124
126
  return logging.getLogger(name)
125
127
 
126
128
 
127
- _SDK_logger = get_SDK_logger()
128
- _SDK_logger.setLevel("DEBUG")
129
+ def _init_logger() -> None:
130
+ level = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
131
+ SDK_logger = get_SDK_logger()
132
+ SDK_logger.setLevel("DEBUG")
133
+
134
+ sh = logging.StreamHandler()
135
+ sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
136
+ sh.setLevel(level)
137
+ SDK_logger.addHandler(sh)
129
138
 
130
- _sh = logging.StreamHandler()
131
- _sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
132
- _sh.setLevel(_SDK_CONSOLE_LOG_LEVEL)
133
- _SDK_logger.addHandler(_sh)
134
139
 
140
+ _init_logger()
135
141
  if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
136
142
  import multiprocessing
137
143