hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/data/demo_data_manifest/__init__.py +3 -0
  3. hpcflow/sdk/__init__.py +4 -1
  4. hpcflow/sdk/app.py +160 -15
  5. hpcflow/sdk/cli.py +14 -0
  6. hpcflow/sdk/cli_common.py +83 -0
  7. hpcflow/sdk/config/__init__.py +4 -0
  8. hpcflow/sdk/config/callbacks.py +25 -2
  9. hpcflow/sdk/config/cli.py +4 -1
  10. hpcflow/sdk/config/config.py +188 -14
  11. hpcflow/sdk/config/config_file.py +91 -3
  12. hpcflow/sdk/config/errors.py +33 -0
  13. hpcflow/sdk/core/__init__.py +2 -0
  14. hpcflow/sdk/core/actions.py +492 -35
  15. hpcflow/sdk/core/cache.py +22 -0
  16. hpcflow/sdk/core/command_files.py +221 -5
  17. hpcflow/sdk/core/commands.py +57 -0
  18. hpcflow/sdk/core/element.py +407 -8
  19. hpcflow/sdk/core/environment.py +92 -0
  20. hpcflow/sdk/core/errors.py +245 -61
  21. hpcflow/sdk/core/json_like.py +72 -14
  22. hpcflow/sdk/core/loop.py +122 -21
  23. hpcflow/sdk/core/loop_cache.py +34 -9
  24. hpcflow/sdk/core/object_list.py +172 -26
  25. hpcflow/sdk/core/parallel.py +14 -0
  26. hpcflow/sdk/core/parameters.py +478 -25
  27. hpcflow/sdk/core/rule.py +31 -1
  28. hpcflow/sdk/core/run_dir_files.py +12 -2
  29. hpcflow/sdk/core/task.py +407 -80
  30. hpcflow/sdk/core/task_schema.py +70 -9
  31. hpcflow/sdk/core/test_utils.py +35 -0
  32. hpcflow/sdk/core/utils.py +101 -4
  33. hpcflow/sdk/core/validation.py +13 -1
  34. hpcflow/sdk/core/workflow.py +316 -96
  35. hpcflow/sdk/core/zarr_io.py +23 -0
  36. hpcflow/sdk/data/__init__.py +13 -0
  37. hpcflow/sdk/demo/__init__.py +3 -0
  38. hpcflow/sdk/helper/__init__.py +3 -0
  39. hpcflow/sdk/helper/cli.py +9 -0
  40. hpcflow/sdk/helper/helper.py +28 -0
  41. hpcflow/sdk/helper/watcher.py +33 -0
  42. hpcflow/sdk/log.py +40 -0
  43. hpcflow/sdk/persistence/__init__.py +14 -4
  44. hpcflow/sdk/persistence/base.py +289 -23
  45. hpcflow/sdk/persistence/json.py +29 -0
  46. hpcflow/sdk/persistence/pending.py +217 -107
  47. hpcflow/sdk/persistence/store_resource.py +58 -2
  48. hpcflow/sdk/persistence/utils.py +8 -0
  49. hpcflow/sdk/persistence/zarr.py +68 -1
  50. hpcflow/sdk/runtime.py +52 -10
  51. hpcflow/sdk/submission/__init__.py +3 -0
  52. hpcflow/sdk/submission/jobscript.py +198 -9
  53. hpcflow/sdk/submission/jobscript_info.py +13 -0
  54. hpcflow/sdk/submission/schedulers/__init__.py +60 -0
  55. hpcflow/sdk/submission/schedulers/direct.py +53 -0
  56. hpcflow/sdk/submission/schedulers/sge.py +45 -7
  57. hpcflow/sdk/submission/schedulers/slurm.py +45 -8
  58. hpcflow/sdk/submission/schedulers/utils.py +4 -0
  59. hpcflow/sdk/submission/shells/__init__.py +11 -1
  60. hpcflow/sdk/submission/shells/base.py +32 -1
  61. hpcflow/sdk/submission/shells/bash.py +36 -1
  62. hpcflow/sdk/submission/shells/os_version.py +18 -6
  63. hpcflow/sdk/submission/shells/powershell.py +22 -0
  64. hpcflow/sdk/submission/submission.py +88 -3
  65. hpcflow/sdk/typing.py +10 -1
  66. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/METADATA +3 -3
  67. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/RECORD +70 -70
  68. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/LICENSE +0 -0
  69. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/WHEEL +0 -0
  70. {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,7 @@
1
+ """
2
+ Persistence model based on writing JSON documents.
3
+ """
4
+
1
5
  from __future__ import annotations
2
6
  from contextlib import contextmanager
3
7
  import copy
@@ -30,6 +34,10 @@ from hpcflow.sdk.persistence.base import update_param_source_dict
30
34
 
31
35
 
32
36
  class JSONPersistentStore(PersistentStore):
37
+ """
38
+ A store that writes JSON files for all its state serialization.
39
+ """
40
+
33
41
  _name = "json"
34
42
  _features = PersistentStoreFeatures(
35
43
  create=True,
@@ -90,6 +98,9 @@ class JSONPersistentStore(PersistentStore):
90
98
  yield md
91
99
 
92
100
  def remove_replaced_dir(self) -> None:
101
+ """
102
+ Remove the directory containing replaced workflow details.
103
+ """
93
104
  with self.using_resource("metadata", "update") as md:
94
105
  if "replaced_workflow" in md:
95
106
  self.remove_path(md["replaced_workflow"], self.fs)
@@ -97,6 +108,9 @@ class JSONPersistentStore(PersistentStore):
97
108
  md["replaced_workflow"] = None
98
109
 
99
110
  def reinstate_replaced_dir(self) -> None:
111
+ """
112
+ Reinstate the directory containing replaced workflow details.
113
+ """
100
114
  with self.using_resource("metadata", "read") as md:
101
115
  if "replaced_workflow" in md:
102
116
  self.logger.debug(
@@ -128,6 +142,9 @@ class JSONPersistentStore(PersistentStore):
128
142
  ts_fmt: str,
129
143
  ts_name_fmt: str,
130
144
  ) -> None:
145
+ """
146
+ Write an empty persistent workflow.
147
+ """
131
148
  fs.mkdir(wk_path)
132
149
  submissions = []
133
150
  parameters = {
@@ -526,17 +543,29 @@ class JSONPersistentStore(PersistentStore):
526
543
  return list(int(i) for i in params["data"].keys())
527
544
 
528
545
  def get_ts_fmt(self):
546
+ """
547
+ Get the format for timestamps.
548
+ """
529
549
  with self.using_resource("metadata", action="read") as md:
530
550
  return md["ts_fmt"]
531
551
 
532
552
  def get_ts_name_fmt(self):
553
+ """
554
+ Get the format for timestamps to use in names.
555
+ """
533
556
  with self.using_resource("metadata", action="read") as md:
534
557
  return md["ts_name_fmt"]
535
558
 
536
559
  def get_creation_info(self):
560
+ """
561
+ Get information about the creation of the workflow.
562
+ """
537
563
  with self.using_resource("metadata", action="read") as md:
538
564
  return copy.deepcopy(md["creation_info"])
539
565
 
540
566
  def get_name(self):
567
+ """
568
+ Get the name of the workflow.
569
+ """
541
570
  with self.using_resource("metadata", action="read") as md:
542
571
  return md["name"]