cfin-hpc-tools 0.2.3b0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. cfin_hpc_tools-0.2.3b0/LICENSE.txt +21 -0
  2. cfin_hpc_tools-0.2.3b0/PKG-INFO +93 -0
  3. cfin_hpc_tools-0.2.3b0/README.md +63 -0
  4. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/PKG-INFO +93 -0
  5. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/SOURCES.txt +139 -0
  6. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/dependency_links.txt +1 -0
  7. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/entry_points.txt +3 -0
  8. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/requires.txt +10 -0
  9. cfin_hpc_tools-0.2.3b0/cfin_hpc_tools.egg-info/top_level.txt +4 -0
  10. cfin_hpc_tools-0.2.3b0/docs/Makefile +20 -0
  11. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/contributing.doctree +0 -0
  12. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/environment.pickle +0 -0
  13. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/example_job_array.doctree +0 -0
  14. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/getting_started.doctree +0 -0
  15. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/index.doctree +0 -0
  16. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/package_management.doctree +0 -0
  17. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/scripts/make_jaf.doctree +0 -0
  18. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/scripts/pyjasub.doctree +0 -0
  19. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/scripts.doctree +0 -0
  20. cfin_hpc_tools-0.2.3b0/docs/build/doctrees/workflow.doctree +0 -0
  21. cfin_hpc_tools-0.2.3b0/docs/build/latex/LICRcyr2utf8.xdy +101 -0
  22. cfin_hpc_tools-0.2.3b0/docs/build/latex/LICRlatin2utf8.xdy +239 -0
  23. cfin_hpc_tools-0.2.3b0/docs/build/latex/LatinRules.xdy +607 -0
  24. cfin_hpc_tools-0.2.3b0/docs/build/latex/Makefile +64 -0
  25. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.aux +115 -0
  26. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.fdb_latexmk +174 -0
  27. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.fls +311 -0
  28. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.idx +0 -0
  29. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.ilg +5 -0
  30. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.ind +0 -0
  31. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.log +902 -0
  32. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.out +41 -0
  33. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.pdf +0 -0
  34. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.tex +1657 -0
  35. cfin_hpc_tools-0.2.3b0/docs/build/latex/hpc-tools.toc +42 -0
  36. cfin_hpc_tools-0.2.3b0/docs/build/latex/latexmkjarc +22 -0
  37. cfin_hpc_tools-0.2.3b0/docs/build/latex/latexmkrc +9 -0
  38. cfin_hpc_tools-0.2.3b0/docs/build/latex/make.bat +31 -0
  39. cfin_hpc_tools-0.2.3b0/docs/build/latex/python.ist +16 -0
  40. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinx.sty +351 -0
  41. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinx.xdy +230 -0
  42. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxhighlight.sty +106 -0
  43. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxhowto.cls +102 -0
  44. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexadmonitions.sty +148 -0
  45. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexcontainers.sty +22 -0
  46. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexgraphics.sty +122 -0
  47. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexindbibtoc.sty +69 -0
  48. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexlists.sty +97 -0
  49. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexliterals.sty +795 -0
  50. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexnumfig.sty +122 -0
  51. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexobjects.sty +200 -0
  52. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexshadowbox.sty +100 -0
  53. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexstyleheadings.sty +83 -0
  54. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexstylepage.sty +79 -0
  55. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatexstyletext.sty +126 -0
  56. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxlatextables.sty +481 -0
  57. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxmanual.cls +128 -0
  58. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxmessages.sty +21 -0
  59. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxoptionsgeometry.sty +54 -0
  60. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxoptionshyperref.sty +35 -0
  61. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxpackagecyrillic.sty +55 -0
  62. cfin_hpc_tools-0.2.3b0/docs/build/latex/sphinxpackagefootnote.sty +396 -0
  63. cfin_hpc_tools-0.2.3b0/docs/make.bat +35 -0
  64. cfin_hpc_tools-0.2.3b0/docs/source/_static/style.css +28 -0
  65. cfin_hpc_tools-0.2.3b0/docs/source/_templates/layout.html +4 -0
  66. cfin_hpc_tools-0.2.3b0/docs/source/conf.py +81 -0
  67. cfin_hpc_tools-0.2.3b0/docs/source/contributing.rst +28 -0
  68. cfin_hpc_tools-0.2.3b0/docs/source/example_job_array.md +305 -0
  69. cfin_hpc_tools-0.2.3b0/docs/source/getting_started.md +79 -0
  70. cfin_hpc_tools-0.2.3b0/docs/source/index.rst +35 -0
  71. cfin_hpc_tools-0.2.3b0/docs/source/package_management.md +2 -0
  72. cfin_hpc_tools-0.2.3b0/docs/source/scripts/make_jaf.rst +46 -0
  73. cfin_hpc_tools-0.2.3b0/docs/source/scripts/pyjasub.rst +182 -0
  74. cfin_hpc_tools-0.2.3b0/docs/source/scripts.rst +12 -0
  75. cfin_hpc_tools-0.2.3b0/docs/source/workflow.rst +33 -0
  76. cfin_hpc_tools-0.2.3b0/hpc_tools/__init__.py +1 -0
  77. cfin_hpc_tools-0.2.3b0/hpc_tools/_version.py +1 -0
  78. cfin_hpc_tools-0.2.3b0/hpc_tools/common.py +139 -0
  79. cfin_hpc_tools-0.2.3b0/hpc_tools/file_helper.py +234 -0
  80. cfin_hpc_tools-0.2.3b0/hpc_tools/make_general_job_array_file.py +612 -0
  81. cfin_hpc_tools-0.2.3b0/hpc_tools/pyjasub.py +1791 -0
  82. cfin_hpc_tools-0.2.3b0/pyproject.toml +75 -0
  83. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/README.md +6 -0
  84. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/env_test.ini +72 -0
  85. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/env_test.sh +6 -0
  86. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/initialise_job.sh +588 -0
  87. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/job_array_tests.sh +460 -0
  88. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/post_run.sh +27 -0
  89. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/pre_run.sh +119 -0
  90. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/profile.sh +61 -0
  91. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/run_job_array.sh +45 -0
  92. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/skeleton_job_array_run.sh +73 -0
  93. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/skeleton_job_array_task.sh +339 -0
  94. cfin_hpc_tools-0.2.3b0/resources/bash_scripts/tar_logs.sh +60 -0
  95. cfin_hpc_tools-0.2.3b0/resources/bin/README.md +2 -0
  96. cfin_hpc_tools-0.2.3b0/resources/bin/csub +180 -0
  97. cfin_hpc_tools-0.2.3b0/resources/bin/task_init.sh +91 -0
  98. cfin_hpc_tools-0.2.3b0/resources/bin/task_setup.sh +285 -0
  99. cfin_hpc_tools-0.2.3b0/resources/bin/wrap +2 -0
  100. cfin_hpc_tools-0.2.3b0/resources/ci_cd/README.md +11 -0
  101. cfin_hpc_tools-0.2.3b0/resources/ci_cd/before_script.sh +20 -0
  102. cfin_hpc_tools-0.2.3b0/resources/ci_cd/pages.sh +28 -0
  103. cfin_hpc_tools-0.2.3b0/resources/ci_cd/pypi-upload.sh +7 -0
  104. cfin_hpc_tools-0.2.3b0/resources/ci_cd/run_docker.sh +57 -0
  105. cfin_hpc_tools-0.2.3b0/resources/ci_cd/run_pages.sh +32 -0
  106. cfin_hpc_tools-0.2.3b0/resources/ci_cd/sphinx.txt +11 -0
  107. cfin_hpc_tools-0.2.3b0/resources/conda/build/py310/LICENSE +674 -0
  108. cfin_hpc_tools-0.2.3b0/resources/conda/build/py310/build.sh +3 -0
  109. cfin_hpc_tools-0.2.3b0/resources/conda/build/py310/meta.yaml +33 -0
  110. cfin_hpc_tools-0.2.3b0/resources/conda/build/py37/LICENSE +674 -0
  111. cfin_hpc_tools-0.2.3b0/resources/conda/build/py37/build.sh +3 -0
  112. cfin_hpc_tools-0.2.3b0/resources/conda/build/py37/meta.yaml +33 -0
  113. cfin_hpc_tools-0.2.3b0/resources/conda/build/py38/LICENSE +674 -0
  114. cfin_hpc_tools-0.2.3b0/resources/conda/build/py38/build.sh +3 -0
  115. cfin_hpc_tools-0.2.3b0/resources/conda/build/py38/meta.yaml +33 -0
  116. cfin_hpc_tools-0.2.3b0/resources/conda/build/py39/LICENSE +674 -0
  117. cfin_hpc_tools-0.2.3b0/resources/conda/build/py39/build.sh +3 -0
  118. cfin_hpc_tools-0.2.3b0/resources/conda/build/py39/meta.yaml +33 -0
  119. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py310/conda_create.yaml +12 -0
  120. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py310/conda_update.yaml +10 -0
  121. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py37/conda_create.yaml +12 -0
  122. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py37/conda_update.yaml +10 -0
  123. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py38/conda_create.yaml +12 -0
  124. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py38/conda_update.yaml +10 -0
  125. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py39/conda_create.yaml +12 -0
  126. cfin_hpc_tools-0.2.3b0/resources/conda/envs/py39/conda_update.yaml +10 -0
  127. cfin_hpc_tools-0.2.3b0/resources/config/md5_sum.ini +32 -0
  128. cfin_hpc_tools-0.2.3b0/resources/example/README.md +305 -0
  129. cfin_hpc_tools-0.2.3b0/resources/example/create_example_files.sh +38 -0
  130. cfin_hpc_tools-0.2.3b0/resources/example/create_job_array.sh +11 -0
  131. cfin_hpc_tools-0.2.3b0/resources/example/example_job.cnf +43 -0
  132. cfin_hpc_tools-0.2.3b0/resources/example/example_task.sh +136 -0
  133. cfin_hpc_tools-0.2.3b0/resources/job_array/README.md +13 -0
  134. cfin_hpc_tools-0.2.3b0/resources/job_array/file-split-job-array.sh +85 -0
  135. cfin_hpc_tools-0.2.3b0/resources/mappings/README.md +6 -0
  136. cfin_hpc_tools-0.2.3b0/resources/mappings/cs_mappings.cnf +14 -0
  137. cfin_hpc_tools-0.2.3b0/resources/mappings/my_mappings.cnf +13 -0
  138. cfin_hpc_tools-0.2.3b0/resources/tasks/README.md +12 -0
  139. cfin_hpc_tools-0.2.3b0/resources/tasks/file-chunk-task.sh +159 -0
  140. cfin_hpc_tools-0.2.3b0/resources/tasks/md5sum.task.sh +54 -0
  141. cfin_hpc_tools-0.2.3b0/setup.cfg +4 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) [year] [fullname]
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: cfin-hpc-tools
3
+ Version: 0.2.3b0
4
+ Summary: Some general grid-engine HPC tools, mainly geared for UCL systems but should be generalisable. Note this is not a UCL endorsed package.
5
+ Author-email: Chris Finan <c.finan@ucl.ac.uk>
6
+ License-Expression: GPL-3.0-or-later
7
+ Project-URL: Homepage, https://cfinan.gitlab.io/cluster
8
+ Project-URL: Repository, https://gitlab.com/cfinan/cluster
9
+ Classifier: Programming Language :: Python
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: Implementation :: CPython
16
+ Classifier: Programming Language :: Python :: Implementation :: PyPy
17
+ Requires-Python: <3.14,>=3.9
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE.txt
20
+ Requires-Dist: terminaltables>=3.1
21
+ Requires-Dist: cfin-pyaddons
22
+ Provides-Extra: dev
23
+ Requires-Dist: pytest; extra == "dev"
24
+ Requires-Dist: pytest-dependency; extra == "dev"
25
+ Requires-Dist: flake8; extra == "dev"
26
+ Requires-Dist: bump2version; extra == "dev"
27
+ Requires-Dist: build; extra == "dev"
28
+ Requires-Dist: twine; extra == "dev"
29
+ Dynamic: license-file
30
+
31
+ # Getting started with hpc-tools
32
+
33
+ __version__: `0.2.3b0`
34
+
35
+ [hpc-tools](https://gitlab.com/cfinan/cluster) is a Python package with some additional bash scripts/configuration files designed to simplify submitting array jobs on a cluster. Whilst it is written with Grid Engine in mind it can probably be re-purposed for other systems (such as LSF) - although this has not been tested in any way. Also, this has only been tested on the University College London (UCL) systems, Myriad and the CS cluster.
36
+
37
+ There is [online](https://cfinan.gitlab.io/cluster/index.html) documentation for hpc-tools and offline PDF documentation can be downloaded [here](https://gitlab.com/cfinan/cluster/-/blob/master/resources/pdf/hpc-tools.pdf).
38
+
39
+ ## Installation instructions
40
+ At present, hpc-tools is undergoing development and no packages exist yet on PyPi. Therefore it is recommended that it is installed in either of the two ways listed below. First, clone this repository and then `cd` to the root of the repository.
41
+
42
+ ### Installation using pip
43
+
44
+ ```
45
+ pip install cfin-hpc-tools
46
+ ```
47
+
48
+ ### Installation using conda
49
+ A conda build is also available for Python v3.7, v3.8 and v3.9 on linux-64.
50
+ ```
51
+ conda install -c cfin cfin-hpc-tools
52
+ ```
53
+
54
+ ### Post Python install
55
+ After installation of hpc-tools, there are some manual setup steps.
56
+
57
+ #### Setup a mapping file
58
+ The main submission script, `pyjasub` works by building job submission commands from user arguments. Therefore there is a mapping step to map between the user aguments and the cluster arguments. This happens internally but how the mapping is performed is controlled by a mapping config file. Examples, can be seen in the `./resources/mappings` directory of the repository and is also shown below:
59
+
60
+ ```
61
+ [mappings]
62
+ array=-t {0}-{1}:{2}
63
+
64
+ # Multiple arguments are separated by commas, such as two memory arguments
65
+ mem=-l hmem={0}, -l vmem={0}
66
+ time=-l h_rt={0}
67
+ nodes=-l hostname={0}
68
+ shell=-S {0}
69
+ out=-o {0}
70
+ err=-e {0}
71
+ batch=-tc {0}
72
+ tmp_size=-l tmpfs={0}
73
+ scratch_size=-l tscratch={0}
74
+ ```
75
+
76
+ This follows a `.ini` format and and the Python formatting characters `{0}`, `{1}` etc... will be substituted with arguments. The mappings should be under the section header `[mappings]`.
77
+
78
+ As a single the mappings file is expected per system, the location of the mappings file should be set in your `~/.bashrc`:
79
+
80
+ ```
81
+ export QSUB_PYTHON_MAPPING="/path/to/mappings.conf"
82
+ ```
83
+
84
+ As the mappings file is a potential security risk, is should only be readable by you. This is checked by `pyjasub` and if any other users have any of `rwx` on the mapping file then `pyjasub` will error out with a `PermissionError`.
85
+
86
+ The abstraction of the cluster arguments has some plusses and minuses, it allows similar commands and job files can be used in different setups. However, this is at the expense of granular control. Having said that `pyjasub` can handle most everyday cluster uses and can probably be extended easily. Currently, `pyjasub` has only really been tested using `GridEngine`, however, it should be able to be adapted for `LSF`.
87
+
88
+ #### Add the `bin` directory to your `PATH`
89
+ There are a number of bash scripts used by `pyjasub`. Whilst, they are not obligatory and the user has full control of the task script, they can be used to perform some setup and cleanup operations. If you make sure that the directory `.resources/bin` is added to your `PATH`, for example, in your `~/.bashrc` file:
90
+
91
+ ```
92
+ export "${PATH}:/path/to/cluster/resources/bin"
93
+ ```
@@ -0,0 +1,63 @@
1
+ # Getting started with hpc-tools
2
+
3
+ __version__: `0.2.3b0`
4
+
5
+ [hpc-tools](https://gitlab.com/cfinan/cluster) is a Python package with some additional bash scripts/configuration files designed to simplify submitting array jobs on a cluster. Whilst it is written with Grid Engine in mind it can probably be re-purposed for other systems (such as LSF) - although this has not been tested in any way. Also, this has only been tested on the University College London (UCL) systems, Myriad and the CS cluster.
6
+
7
+ There is [online](https://cfinan.gitlab.io/cluster/index.html) documentation for hpc-tools and offline PDF documentation can be downloaded [here](https://gitlab.com/cfinan/cluster/-/blob/master/resources/pdf/hpc-tools.pdf).
8
+
9
+ ## Installation instructions
10
+ At present, hpc-tools is undergoing development and no packages exist yet on PyPi. Therefore it is recommended that it is installed in either of the two ways listed below. First, clone this repository and then `cd` to the root of the repository.
11
+
12
+ ### Installation using pip
13
+
14
+ ```
15
+ pip install cfin-hpc-tools
16
+ ```
17
+
18
+ ### Installation using conda
19
+ A conda build is also available for Python v3.7, v3.8 and v3.9 on linux-64.
20
+ ```
21
+ conda install -c cfin cfin-hpc-tools
22
+ ```
23
+
24
+ ### Post Python install
25
+ After installation of hpc-tools, there are some manual setup steps.
26
+
27
+ #### Setup a mapping file
28
+ The main submission script, `pyjasub` works by building job submission commands from user arguments. Therefore there is a mapping step to map between the user aguments and the cluster arguments. This happens internally but how the mapping is performed is controlled by a mapping config file. Examples, can be seen in the `./resources/mappings` directory of the repository and is also shown below:
29
+
30
+ ```
31
+ [mappings]
32
+ array=-t {0}-{1}:{2}
33
+
34
+ # Multiple arguments are separated by commas, such as two memory arguments
35
+ mem=-l hmem={0}, -l vmem={0}
36
+ time=-l h_rt={0}
37
+ nodes=-l hostname={0}
38
+ shell=-S {0}
39
+ out=-o {0}
40
+ err=-e {0}
41
+ batch=-tc {0}
42
+ tmp_size=-l tmpfs={0}
43
+ scratch_size=-l tscratch={0}
44
+ ```
45
+
46
+ This follows a `.ini` format and and the Python formatting characters `{0}`, `{1}` etc... will be substituted with arguments. The mappings should be under the section header `[mappings]`.
47
+
48
+ As a single the mappings file is expected per system, the location of the mappings file should be set in your `~/.bashrc`:
49
+
50
+ ```
51
+ export QSUB_PYTHON_MAPPING="/path/to/mappings.conf"
52
+ ```
53
+
54
+ As the mappings file is a potential security risk, is should only be readable by you. This is checked by `pyjasub` and if any other users have any of `rwx` on the mapping file then `pyjasub` will error out with a `PermissionError`.
55
+
56
+ The abstraction of the cluster arguments has some plusses and minuses, it allows similar commands and job files can be used in different setups. However, this is at the expense of granular control. Having said that `pyjasub` can handle most everyday cluster uses and can probably be extended easily. Currently, `pyjasub` has only really been tested using `GridEngine`, however, it should be able to be adapted for `LSF`.
57
+
58
+ #### Add the `bin` directory to your `PATH`
59
+ There are a number of bash scripts used by `pyjasub`. Whilst, they are not obligatory and the user has full control of the task script, they can be used to perform some setup and cleanup operations. If you make sure that the directory `.resources/bin` is added to your `PATH`, for example, in your `~/.bashrc` file:
60
+
61
+ ```
62
+ export "${PATH}:/path/to/cluster/resources/bin"
63
+ ```
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: cfin-hpc-tools
3
+ Version: 0.2.3b0
4
+ Summary: Some general grid-engine HPC tools, mainly geared for UCL systems but should be generalisable. Note this is not a UCL endorsed package.
5
+ Author-email: Chris Finan <c.finan@ucl.ac.uk>
6
+ License-Expression: GPL-3.0-or-later
7
+ Project-URL: Homepage, https://cfinan.gitlab.io/cluster
8
+ Project-URL: Repository, https://gitlab.com/cfinan/cluster
9
+ Classifier: Programming Language :: Python
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: Implementation :: CPython
16
+ Classifier: Programming Language :: Python :: Implementation :: PyPy
17
+ Requires-Python: <3.14,>=3.9
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE.txt
20
+ Requires-Dist: terminaltables>=3.1
21
+ Requires-Dist: cfin-pyaddons
22
+ Provides-Extra: dev
23
+ Requires-Dist: pytest; extra == "dev"
24
+ Requires-Dist: pytest-dependency; extra == "dev"
25
+ Requires-Dist: flake8; extra == "dev"
26
+ Requires-Dist: bump2version; extra == "dev"
27
+ Requires-Dist: build; extra == "dev"
28
+ Requires-Dist: twine; extra == "dev"
29
+ Dynamic: license-file
30
+
31
+ # Getting started with hpc-tools
32
+
33
+ __version__: `0.2.3b0`
34
+
35
+ [hpc-tools](https://gitlab.com/cfinan/cluster) is a Python package with some additional bash scripts/configuration files designed to simplify submitting array jobs on a cluster. Whilst it is written with Grid Engine in mind it can probably be re-purposed for other systems (such as LSF) - although this has not been tested in any way. Also, this has only been tested on the University College London (UCL) systems, Myriad and the CS cluster.
36
+
37
+ There is [online](https://cfinan.gitlab.io/cluster/index.html) documentation for hpc-tools and offline PDF documentation can be downloaded [here](https://gitlab.com/cfinan/cluster/-/blob/master/resources/pdf/hpc-tools.pdf).
38
+
39
+ ## Installation instructions
40
+ At present, hpc-tools is undergoing development and no packages exist yet on PyPi. Therefore it is recommended that it is installed in either of the two ways listed below. First, clone this repository and then `cd` to the root of the repository.
41
+
42
+ ### Installation using pip
43
+
44
+ ```
45
+ pip install cfin-hpc-tools
46
+ ```
47
+
48
+ ### Installation using conda
49
+ A conda build is also available for Python v3.7, v3.8 and v3.9 on linux-64.
50
+ ```
51
+ conda install -c cfin cfin-hpc-tools
52
+ ```
53
+
54
+ ### Post Python install
55
+ After installation of hpc-tools, there are some manual setup steps.
56
+
57
+ #### Setup a mapping file
58
+ The main submission script, `pyjasub` works by building job submission commands from user arguments. Therefore there is a mapping step to map between the user aguments and the cluster arguments. This happens internally but how the mapping is performed is controlled by a mapping config file. Examples, can be seen in the `./resources/mappings` directory of the repository and is also shown below:
59
+
60
+ ```
61
+ [mappings]
62
+ array=-t {0}-{1}:{2}
63
+
64
+ # Multiple arguments are separated by commas, such as two memory arguments
65
+ mem=-l hmem={0}, -l vmem={0}
66
+ time=-l h_rt={0}
67
+ nodes=-l hostname={0}
68
+ shell=-S {0}
69
+ out=-o {0}
70
+ err=-e {0}
71
+ batch=-tc {0}
72
+ tmp_size=-l tmpfs={0}
73
+ scratch_size=-l tscratch={0}
74
+ ```
75
+
76
+ This follows a `.ini` format and and the Python formatting characters `{0}`, `{1}` etc... will be substituted with arguments. The mappings should be under the section header `[mappings]`.
77
+
78
+ As a single the mappings file is expected per system, the location of the mappings file should be set in your `~/.bashrc`:
79
+
80
+ ```
81
+ export QSUB_PYTHON_MAPPING="/path/to/mappings.conf"
82
+ ```
83
+
84
+ As the mappings file is a potential security risk, is should only be readable by you. This is checked by `pyjasub` and if any other users have any of `rwx` on the mapping file then `pyjasub` will error out with a `PermissionError`.
85
+
86
+ The abstraction of the cluster arguments has some plusses and minuses, it allows similar commands and job files can be used in different setups. However, this is at the expense of granular control. Having said that `pyjasub` can handle most everyday cluster uses and can probably be extended easily. Currently, `pyjasub` has only really been tested using `GridEngine`, however, it should be able to be adapted for `LSF`.
87
+
88
+ #### Add the `bin` directory to your `PATH`
89
+ There are a number of bash scripts used by `pyjasub`. Whilst, they are not obligatory and the user has full control of the task script, they can be used to perform some setup and cleanup operations. If you make sure that the directory `.resources/bin` is added to your `PATH`, for example, in your `~/.bashrc` file:
90
+
91
+ ```
92
+ export "${PATH}:/path/to/cluster/resources/bin"
93
+ ```
@@ -0,0 +1,139 @@
1
+ LICENSE.txt
2
+ README.md
3
+ pyproject.toml
4
+ cfin_hpc_tools.egg-info/PKG-INFO
5
+ cfin_hpc_tools.egg-info/SOURCES.txt
6
+ cfin_hpc_tools.egg-info/dependency_links.txt
7
+ cfin_hpc_tools.egg-info/entry_points.txt
8
+ cfin_hpc_tools.egg-info/requires.txt
9
+ cfin_hpc_tools.egg-info/top_level.txt
10
+ docs/Makefile
11
+ docs/make.bat
12
+ docs/build/doctrees/contributing.doctree
13
+ docs/build/doctrees/environment.pickle
14
+ docs/build/doctrees/example_job_array.doctree
15
+ docs/build/doctrees/getting_started.doctree
16
+ docs/build/doctrees/index.doctree
17
+ docs/build/doctrees/package_management.doctree
18
+ docs/build/doctrees/scripts.doctree
19
+ docs/build/doctrees/workflow.doctree
20
+ docs/build/doctrees/scripts/make_jaf.doctree
21
+ docs/build/doctrees/scripts/pyjasub.doctree
22
+ docs/build/latex/LICRcyr2utf8.xdy
23
+ docs/build/latex/LICRlatin2utf8.xdy
24
+ docs/build/latex/LatinRules.xdy
25
+ docs/build/latex/Makefile
26
+ docs/build/latex/hpc-tools.aux
27
+ docs/build/latex/hpc-tools.fdb_latexmk
28
+ docs/build/latex/hpc-tools.fls
29
+ docs/build/latex/hpc-tools.idx
30
+ docs/build/latex/hpc-tools.ilg
31
+ docs/build/latex/hpc-tools.ind
32
+ docs/build/latex/hpc-tools.log
33
+ docs/build/latex/hpc-tools.out
34
+ docs/build/latex/hpc-tools.pdf
35
+ docs/build/latex/hpc-tools.tex
36
+ docs/build/latex/hpc-tools.toc
37
+ docs/build/latex/latexmkjarc
38
+ docs/build/latex/latexmkrc
39
+ docs/build/latex/make.bat
40
+ docs/build/latex/python.ist
41
+ docs/build/latex/sphinx.sty
42
+ docs/build/latex/sphinx.xdy
43
+ docs/build/latex/sphinxhighlight.sty
44
+ docs/build/latex/sphinxhowto.cls
45
+ docs/build/latex/sphinxlatexadmonitions.sty
46
+ docs/build/latex/sphinxlatexcontainers.sty
47
+ docs/build/latex/sphinxlatexgraphics.sty
48
+ docs/build/latex/sphinxlatexindbibtoc.sty
49
+ docs/build/latex/sphinxlatexlists.sty
50
+ docs/build/latex/sphinxlatexliterals.sty
51
+ docs/build/latex/sphinxlatexnumfig.sty
52
+ docs/build/latex/sphinxlatexobjects.sty
53
+ docs/build/latex/sphinxlatexshadowbox.sty
54
+ docs/build/latex/sphinxlatexstyleheadings.sty
55
+ docs/build/latex/sphinxlatexstylepage.sty
56
+ docs/build/latex/sphinxlatexstyletext.sty
57
+ docs/build/latex/sphinxlatextables.sty
58
+ docs/build/latex/sphinxmanual.cls
59
+ docs/build/latex/sphinxmessages.sty
60
+ docs/build/latex/sphinxoptionsgeometry.sty
61
+ docs/build/latex/sphinxoptionshyperref.sty
62
+ docs/build/latex/sphinxpackagecyrillic.sty
63
+ docs/build/latex/sphinxpackagefootnote.sty
64
+ docs/source/conf.py
65
+ docs/source/contributing.rst
66
+ docs/source/example_job_array.md
67
+ docs/source/getting_started.md
68
+ docs/source/index.rst
69
+ docs/source/package_management.md
70
+ docs/source/scripts.rst
71
+ docs/source/workflow.rst
72
+ docs/source/_static/style.css
73
+ docs/source/_templates/layout.html
74
+ docs/source/scripts/make_jaf.rst
75
+ docs/source/scripts/pyjasub.rst
76
+ hpc_tools/__init__.py
77
+ hpc_tools/_version.py
78
+ hpc_tools/common.py
79
+ hpc_tools/file_helper.py
80
+ hpc_tools/make_general_job_array_file.py
81
+ hpc_tools/pyjasub.py
82
+ resources/bash_scripts/README.md
83
+ resources/bash_scripts/env_test.ini
84
+ resources/bash_scripts/env_test.sh
85
+ resources/bash_scripts/initialise_job.sh
86
+ resources/bash_scripts/job_array_tests.sh
87
+ resources/bash_scripts/post_run.sh
88
+ resources/bash_scripts/pre_run.sh
89
+ resources/bash_scripts/profile.sh
90
+ resources/bash_scripts/run_job_array.sh
91
+ resources/bash_scripts/skeleton_job_array_run.sh
92
+ resources/bash_scripts/skeleton_job_array_task.sh
93
+ resources/bash_scripts/tar_logs.sh
94
+ resources/bin/README.md
95
+ resources/bin/csub
96
+ resources/bin/task_init.sh
97
+ resources/bin/task_setup.sh
98
+ resources/bin/wrap
99
+ resources/ci_cd/README.md
100
+ resources/ci_cd/before_script.sh
101
+ resources/ci_cd/pages.sh
102
+ resources/ci_cd/pypi-upload.sh
103
+ resources/ci_cd/run_docker.sh
104
+ resources/ci_cd/run_pages.sh
105
+ resources/ci_cd/sphinx.txt
106
+ resources/conda/build/py310/LICENSE
107
+ resources/conda/build/py310/build.sh
108
+ resources/conda/build/py310/meta.yaml
109
+ resources/conda/build/py37/LICENSE
110
+ resources/conda/build/py37/build.sh
111
+ resources/conda/build/py37/meta.yaml
112
+ resources/conda/build/py38/LICENSE
113
+ resources/conda/build/py38/build.sh
114
+ resources/conda/build/py38/meta.yaml
115
+ resources/conda/build/py39/LICENSE
116
+ resources/conda/build/py39/build.sh
117
+ resources/conda/build/py39/meta.yaml
118
+ resources/conda/envs/py310/conda_create.yaml
119
+ resources/conda/envs/py310/conda_update.yaml
120
+ resources/conda/envs/py37/conda_create.yaml
121
+ resources/conda/envs/py37/conda_update.yaml
122
+ resources/conda/envs/py38/conda_create.yaml
123
+ resources/conda/envs/py38/conda_update.yaml
124
+ resources/conda/envs/py39/conda_create.yaml
125
+ resources/conda/envs/py39/conda_update.yaml
126
+ resources/config/md5_sum.ini
127
+ resources/example/README.md
128
+ resources/example/create_example_files.sh
129
+ resources/example/create_job_array.sh
130
+ resources/example/example_job.cnf
131
+ resources/example/example_task.sh
132
+ resources/job_array/README.md
133
+ resources/job_array/file-split-job-array.sh
134
+ resources/mappings/README.md
135
+ resources/mappings/cs_mappings.cnf
136
+ resources/mappings/my_mappings.cnf
137
+ resources/tasks/README.md
138
+ resources/tasks/file-chunk-task.sh
139
+ resources/tasks/md5sum.task.sh
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ make-jaf = hpc_tools.make_general_job_array_file:main
3
+ pyjasub = hpc_tools.pyjasub:main
@@ -0,0 +1,10 @@
1
+ terminaltables>=3.1
2
+ cfin-pyaddons
3
+
4
+ [dev]
5
+ pytest
6
+ pytest-dependency
7
+ flake8
8
+ bump2version
9
+ build
10
+ twine
@@ -0,0 +1,4 @@
1
+ dist
2
+ docs
3
+ hpc_tools
4
+ resources
@@ -0,0 +1,20 @@
1
+ # Minimal makefile for Sphinx documentation
2
+ #
3
+
4
+ # You can set these variables from the command line, and also
5
+ # from the environment for the first two.
6
+ SPHINXOPTS ?=
7
+ SPHINXBUILD ?= sphinx-build
8
+ SOURCEDIR = source
9
+ BUILDDIR = build
10
+
11
+ # Put it first so that "make" without argument is like "make help".
12
+ help:
13
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14
+
15
+ .PHONY: help Makefile
16
+
17
+ # Catch-all target: route all unknown targets to Sphinx using the new
18
+ # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19
+ %: Makefile
20
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@@ -0,0 +1,101 @@
1
+ ;; -*- coding: utf-8; mode: Lisp; -*-
2
+ ;; style file for xindy
3
+ ;; filename: LICRcyr2utf8.xdy
4
+ ;; description: style file for xindy which maps back LaTeX Internal
5
+ ;; Character Representation of Cyrillic to utf-8
6
+ ;; usage: for use with pdflatex produced .idx files.
7
+ ;; Contributed by the Sphinx team, July 2018.
8
+ (merge-rule "\IeC {\'\CYRG }" "Ѓ" :string)
9
+ (merge-rule "\IeC {\'\CYRK }" "Ќ" :string)
10
+ (merge-rule "\IeC {\'\cyrg }" "ѓ" :string)
11
+ (merge-rule "\IeC {\'\cyrk }" "ќ" :string)
12
+ (merge-rule "\IeC {\CYRA }" "А" :string)
13
+ (merge-rule "\IeC {\CYRB }" "Б" :string)
14
+ (merge-rule "\IeC {\CYRC }" "Ц" :string)
15
+ (merge-rule "\IeC {\CYRCH }" "Ч" :string)
16
+ (merge-rule "\IeC {\CYRD }" "Д" :string)
17
+ (merge-rule "\IeC {\CYRDJE }" "Ђ" :string)
18
+ (merge-rule "\IeC {\CYRDZE }" "Ѕ" :string)
19
+ (merge-rule "\IeC {\CYRDZHE }" "Џ" :string)
20
+ (merge-rule "\IeC {\CYRE }" "Е" :string)
21
+ (merge-rule "\IeC {\CYREREV }" "Э" :string)
22
+ (merge-rule "\IeC {\CYRERY }" "Ы" :string)
23
+ (merge-rule "\IeC {\CYRF }" "Ф" :string)
24
+ (merge-rule "\IeC {\CYRG }" "Г" :string)
25
+ (merge-rule "\IeC {\CYRGUP }" "Ґ" :string)
26
+ (merge-rule "\IeC {\CYRH }" "Х" :string)
27
+ (merge-rule "\IeC {\CYRHRDSN }" "Ъ" :string)
28
+ (merge-rule "\IeC {\CYRI }" "И" :string)
29
+ (merge-rule "\IeC {\CYRIE }" "Є" :string)
30
+ (merge-rule "\IeC {\CYRII }" "І" :string)
31
+ (merge-rule "\IeC {\CYRISHRT }" "Й" :string)
32
+ (merge-rule "\IeC {\CYRJE }" "Ј" :string)
33
+ (merge-rule "\IeC {\CYRK }" "К" :string)
34
+ (merge-rule "\IeC {\CYRL }" "Л" :string)
35
+ (merge-rule "\IeC {\CYRLJE }" "Љ" :string)
36
+ (merge-rule "\IeC {\CYRM }" "М" :string)
37
+ (merge-rule "\IeC {\CYRN }" "Н" :string)
38
+ (merge-rule "\IeC {\CYRNJE }" "Њ" :string)
39
+ (merge-rule "\IeC {\CYRO }" "О" :string)
40
+ (merge-rule "\IeC {\CYRP }" "П" :string)
41
+ (merge-rule "\IeC {\CYRR }" "Р" :string)
42
+ (merge-rule "\IeC {\CYRS }" "С" :string)
43
+ (merge-rule "\IeC {\CYRSFTSN }" "Ь" :string)
44
+ (merge-rule "\IeC {\CYRSH }" "Ш" :string)
45
+ (merge-rule "\IeC {\CYRSHCH }" "Щ" :string)
46
+ (merge-rule "\IeC {\CYRT }" "Т" :string)
47
+ (merge-rule "\IeC {\CYRTSHE }" "Ћ" :string)
48
+ (merge-rule "\IeC {\CYRU }" "У" :string)
49
+ (merge-rule "\IeC {\CYRUSHRT }" "Ў" :string)
50
+ (merge-rule "\IeC {\CYRV }" "В" :string)
51
+ (merge-rule "\IeC {\CYRYA }" "Я" :string)
52
+ (merge-rule "\IeC {\CYRYI }" "Ї" :string)
53
+ (merge-rule "\IeC {\CYRYO }" "Ё" :string)
54
+ (merge-rule "\IeC {\CYRYU }" "Ю" :string)
55
+ (merge-rule "\IeC {\CYRZ }" "З" :string)
56
+ (merge-rule "\IeC {\CYRZH }" "Ж" :string)
57
+ (merge-rule "\IeC {\cyra }" "а" :string)
58
+ (merge-rule "\IeC {\cyrb }" "б" :string)
59
+ (merge-rule "\IeC {\cyrc }" "ц" :string)
60
+ (merge-rule "\IeC {\cyrch }" "ч" :string)
61
+ (merge-rule "\IeC {\cyrd }" "д" :string)
62
+ (merge-rule "\IeC {\cyrdje }" "ђ" :string)
63
+ (merge-rule "\IeC {\cyrdze }" "ѕ" :string)
64
+ (merge-rule "\IeC {\cyrdzhe }" "џ" :string)
65
+ (merge-rule "\IeC {\cyre }" "е" :string)
66
+ (merge-rule "\IeC {\cyrerev }" "э" :string)
67
+ (merge-rule "\IeC {\cyrery }" "ы" :string)
68
+ (merge-rule "\IeC {\cyrf }" "ф" :string)
69
+ (merge-rule "\IeC {\cyrg }" "г" :string)
70
+ (merge-rule "\IeC {\cyrgup }" "ґ" :string)
71
+ (merge-rule "\IeC {\cyrh }" "х" :string)
72
+ (merge-rule "\IeC {\cyrhrdsn }" "ъ" :string)
73
+ (merge-rule "\IeC {\cyri }" "и" :string)
74
+ (merge-rule "\IeC {\cyrie }" "є" :string)
75
+ (merge-rule "\IeC {\cyrii }" "і" :string)
76
+ (merge-rule "\IeC {\cyrishrt }" "й" :string)
77
+ (merge-rule "\IeC {\cyrje }" "ј" :string)
78
+ (merge-rule "\IeC {\cyrk }" "к" :string)
79
+ (merge-rule "\IeC {\cyrl }" "л" :string)
80
+ (merge-rule "\IeC {\cyrlje }" "љ" :string)
81
+ (merge-rule "\IeC {\cyrm }" "м" :string)
82
+ (merge-rule "\IeC {\cyrn }" "н" :string)
83
+ (merge-rule "\IeC {\cyrnje }" "њ" :string)
84
+ (merge-rule "\IeC {\cyro }" "о" :string)
85
+ (merge-rule "\IeC {\cyrp }" "п" :string)
86
+ (merge-rule "\IeC {\cyrr }" "р" :string)
87
+ (merge-rule "\IeC {\cyrs }" "с" :string)
88
+ (merge-rule "\IeC {\cyrsftsn }" "ь" :string)
89
+ (merge-rule "\IeC {\cyrsh }" "ш" :string)
90
+ (merge-rule "\IeC {\cyrshch }" "щ" :string)
91
+ (merge-rule "\IeC {\cyrt }" "т" :string)
92
+ (merge-rule "\IeC {\cyrtshe }" "ћ" :string)
93
+ (merge-rule "\IeC {\cyru }" "у" :string)
94
+ (merge-rule "\IeC {\cyrushrt }" "ў" :string)
95
+ (merge-rule "\IeC {\cyrv }" "в" :string)
96
+ (merge-rule "\IeC {\cyrya }" "я" :string)
97
+ (merge-rule "\IeC {\cyryi }" "ї" :string)
98
+ (merge-rule "\IeC {\cyryo }" "ё" :string)
99
+ (merge-rule "\IeC {\cyryu }" "ю" :string)
100
+ (merge-rule "\IeC {\cyrz }" "з" :string)
101
+ (merge-rule "\IeC {\cyrzh }" "ж" :string)