shexec 2.4.0__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shexec-2.4.0/shexec.egg-info → shexec-2.6.0}/PKG-INFO +18 -14
- {shexec-2.4.0 → shexec-2.6.0}/README.rst +14 -10
- {shexec-2.4.0 → shexec-2.6.0}/pyproject.toml +1 -1
- {shexec-2.4.0 → shexec-2.6.0}/shexec/__init__.py +5 -2
- {shexec-2.4.0 → shexec-2.6.0}/shexec/models.py +57 -2
- {shexec-2.4.0 → shexec-2.6.0}/shexec/plugin.py +28 -12
- {shexec-2.4.0 → shexec-2.6.0}/shexec/shexec.py +40 -5
- {shexec-2.4.0 → shexec-2.6.0/shexec.egg-info}/PKG-INFO +18 -14
- {shexec-2.4.0 → shexec-2.6.0}/shexec.egg-info/requires.txt +3 -3
- {shexec-2.4.0 → shexec-2.6.0}/tests/test_plugin.py +47 -0
- {shexec-2.4.0 → shexec-2.6.0}/LICENSE +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/setup.cfg +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/shexec/__main__.py +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/shexec.egg-info/SOURCES.txt +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/shexec.egg-info/dependency_links.txt +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/shexec.egg-info/entry_points.txt +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/shexec.egg-info/top_level.txt +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/tests/test_cli.py +0 -0
- {shexec-2.4.0 → shexec-2.6.0}/tests/test_shexec.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shexec
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Run a shell-type command over every file of an input tree: a ChRIS DS plugin on pfdo_run
|
|
5
5
|
Home-page: https://github.com/FNNDSC/pl-shexec
|
|
6
6
|
Author: FNNDSC
|
|
@@ -18,9 +18,9 @@ Requires-Python: >=3.11
|
|
|
18
18
|
Description-Content-Type: text/x-rst
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: chris_plugin>=0.4.0
|
|
21
|
-
Requires-Dist: pfdo>=6.
|
|
22
|
-
Requires-Dist: pfdo_run>=4.
|
|
23
|
-
Requires-Dist: hostlib>=1.
|
|
21
|
+
Requires-Dist: pfdo>=6.4.0
|
|
22
|
+
Requires-Dist: pfdo_run>=4.4.0
|
|
23
|
+
Requires-Dist: hostlib>=1.3.0
|
|
24
24
|
Requires-Dist: pflog>=2.0.0
|
|
25
25
|
Provides-Extra: dev
|
|
26
26
|
Requires-Dist: pytest>=8; extra == "dev"
|
|
@@ -41,13 +41,13 @@ pl-shexec
|
|
|
41
41
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
42
42
|
/incoming /outgoing
|
|
43
43
|
|
|
44
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
44
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
45
45
|
|
|
46
46
|
.. code-block:: bash
|
|
47
47
|
|
|
48
|
-
shexec --analyzeFileIndex
|
|
48
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
49
49
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
50
|
-
-s %{
|
|
50
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
51
51
|
/incoming /outgoing
|
|
52
52
|
|
|
53
53
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -57,12 +57,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
57
57
|
The template
|
|
58
58
|
------------
|
|
59
59
|
|
|
60
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
60
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
61
61
|
|
|
62
62
|
========================== ==============================================
|
|
63
63
|
``%{inputWorkingDir}`` the input directory being processed
|
|
64
64
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
65
65
|
``%{inputWorkingFile}`` the file being processed
|
|
66
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
67
|
+
name for a volume alone in it or a ``.dcm``
|
|
68
|
+
series; for a file beside other volumes, its
|
|
69
|
+
name less its extension (``scan01.nii.gz``)
|
|
66
70
|
========================== ==============================================
|
|
67
71
|
|
|
68
72
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -92,9 +96,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
92
96
|
Selecting files and directories
|
|
93
97
|
-------------------------------
|
|
94
98
|
|
|
95
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
99
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
96
100
|
|
|
97
|
-
Directories run in parallel under ``--threads
|
|
101
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
98
102
|
|
|
99
103
|
Exit status
|
|
100
104
|
-----------
|
|
@@ -108,7 +112,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
108
112
|
|
|
109
113
|
- the image carries the program, with whatever licence or data it needs;
|
|
110
114
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
111
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
115
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
112
116
|
|
|
113
117
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
114
118
|
|
|
@@ -126,7 +130,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
126
130
|
|
|
127
131
|
RECON_ALL: str = (
|
|
128
132
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
129
|
-
"-s %{
|
|
133
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
130
134
|
)
|
|
131
135
|
|
|
132
136
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -135,14 +139,14 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
135
139
|
title="FreeSurfer recon-all over every subject",
|
|
136
140
|
minMemory="8Gi",
|
|
137
141
|
minCpu="4000m",
|
|
138
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
142
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
139
143
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
140
144
|
description=DESCRIPTION,
|
|
141
145
|
examples=EXAMPLES,
|
|
142
146
|
))
|
|
143
147
|
main = PLUGIN.main
|
|
144
148
|
|
|
145
|
-
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(
|
|
149
|
+
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(...)`` removes them from its parser, its ChRIS description and its manual, and fixes their values. A flag that suits a shell but not ChRIS goes in ``chrisHidden`` instead: it stays on the command line and in the manual and is left out of the description ChRIS reads, so in ChRIS it keeps its default. ``jobMemory="16Gi"`` says how much memory one job needs at most; ``--threads N`` is then capped at the memory the process may use, its container's or Slurm job's limit or the machine's available memory as ``hostlib.memory_limit`` reads it, divided by that figure, and the cap is said on standard error. pl-freesurfer uses both: one ``recon-all`` at a time in a 32Gi ChRIS container, several at once on a workstation or a Slurm node with the memory for them. ``plugin_build`` returns the parser with those defaults applied, the manual, and ``main`` already wrapped in pflog's ``timed`` and ``chris_plugin``, ready to be the console script. The engine, the flags, the records and the exit status are not in the spec, because they are the same for every plugin on the stack, and a fix to them reaches every plugin on its next build.
|
|
146
150
|
|
|
147
151
|
The rest of pl-freesurfer is a ``pyproject.toml`` naming ``chris_plugin`` and ``shexec`` as its dependencies, the first so that ``chris_plugin_info`` finds the plugin, a ``setup.cfg``, a Dockerfile that starts from the FreeSurfer image and installs the package, tests, and docs. pl-shexec itself is the first plugin built this way; its ``__main__`` is the same call with an empty default command. The pf* stack's ``docs/plugin.adoc`` has the template.
|
|
148
152
|
|
|
@@ -12,13 +12,13 @@ pl-shexec
|
|
|
12
12
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
13
13
|
/incoming /outgoing
|
|
14
14
|
|
|
15
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
15
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
16
16
|
|
|
17
17
|
.. code-block:: bash
|
|
18
18
|
|
|
19
|
-
shexec --analyzeFileIndex
|
|
19
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
20
20
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
21
|
-
-s %{
|
|
21
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
22
22
|
/incoming /outgoing
|
|
23
23
|
|
|
24
24
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -28,12 +28,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
28
28
|
The template
|
|
29
29
|
------------
|
|
30
30
|
|
|
31
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
31
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
32
32
|
|
|
33
33
|
========================== ==============================================
|
|
34
34
|
``%{inputWorkingDir}`` the input directory being processed
|
|
35
35
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
36
36
|
``%{inputWorkingFile}`` the file being processed
|
|
37
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
38
|
+
name for a volume alone in it or a ``.dcm``
|
|
39
|
+
series; for a file beside other volumes, its
|
|
40
|
+
name less its extension (``scan01.nii.gz``)
|
|
37
41
|
========================== ==============================================
|
|
38
42
|
|
|
39
43
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -63,9 +67,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
63
67
|
Selecting files and directories
|
|
64
68
|
-------------------------------
|
|
65
69
|
|
|
66
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
70
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
67
71
|
|
|
68
|
-
Directories run in parallel under ``--threads
|
|
72
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
69
73
|
|
|
70
74
|
Exit status
|
|
71
75
|
-----------
|
|
@@ -79,7 +83,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
79
83
|
|
|
80
84
|
- the image carries the program, with whatever licence or data it needs;
|
|
81
85
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
82
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
86
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
83
87
|
|
|
84
88
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
85
89
|
|
|
@@ -97,7 +101,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
97
101
|
|
|
98
102
|
RECON_ALL: str = (
|
|
99
103
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
100
|
-
"-s %{
|
|
104
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
101
105
|
)
|
|
102
106
|
|
|
103
107
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -106,14 +110,14 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
106
110
|
title="FreeSurfer recon-all over every subject",
|
|
107
111
|
minMemory="8Gi",
|
|
108
112
|
minCpu="4000m",
|
|
109
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
113
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
110
114
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
111
115
|
description=DESCRIPTION,
|
|
112
116
|
examples=EXAMPLES,
|
|
113
117
|
))
|
|
114
118
|
main = PLUGIN.main
|
|
115
119
|
|
|
116
|
-
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(
|
|
120
|
+
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(...)`` removes them from its parser, its ChRIS description and its manual, and fixes their values. A flag that suits a shell but not ChRIS goes in ``chrisHidden`` instead: it stays on the command line and in the manual and is left out of the description ChRIS reads, so in ChRIS it keeps its default. ``jobMemory="16Gi"`` says how much memory one job needs at most; ``--threads N`` is then capped at the memory the process may use, its container's or Slurm job's limit or the machine's available memory as ``hostlib.memory_limit`` reads it, divided by that figure, and the cap is said on standard error. pl-freesurfer uses both: one ``recon-all`` at a time in a 32Gi ChRIS container, several at once on a workstation or a Slurm node with the memory for them. ``plugin_build`` returns the parser with those defaults applied, the manual, and ``main`` already wrapped in pflog's ``timed`` and ``chris_plugin``, ready to be the console script. The engine, the flags, the records and the exit status are not in the spec, because they are the same for every plugin on the stack, and a fix to them reaches every plugin on its next build.
|
|
117
121
|
|
|
118
122
|
The rest of pl-freesurfer is a ``pyproject.toml`` naming ``chris_plugin`` and ``shexec`` as its dependencies, the first so that ``chris_plugin_info`` finds the plugin, a ``setup.cfg``, a Dockerfile that starts from the FreeSurfer image and installs the package, tests, and docs. pl-shexec itself is the first plugin built this way; its ``__main__`` is the same call with an empty default command. The pf* stack's ``docs/plugin.adoc`` has the template.
|
|
119
123
|
|
|
@@ -13,7 +13,7 @@ authors = [{ email = "dev@babyMRI.org" }]
|
|
|
13
13
|
# fields, which the [project] table does not emit; setup.cfg supplies them, and the
|
|
14
14
|
# author name lives there alone so Author-email stays a bare address.
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
|
-
dependencies = ["chris_plugin>=0.4.0", "pfdo>=6.
|
|
16
|
+
dependencies = ["chris_plugin>=0.4.0", "pfdo>=6.4.0", "pfdo_run>=4.4.0", "hostlib>=1.3.0", "pflog>=2.0.0"]
|
|
17
17
|
classifiers = [
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
@@ -14,7 +14,7 @@ Dependencies:
|
|
|
14
14
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
|
-
__version__: str = "2.
|
|
17
|
+
__version__: str = "2.6.0"
|
|
18
18
|
|
|
19
19
|
from shexec.models import (
|
|
20
20
|
DirNode,
|
|
@@ -28,9 +28,10 @@ from shexec.models import (
|
|
|
28
28
|
ShExecOptions,
|
|
29
29
|
Span,
|
|
30
30
|
StageResult,
|
|
31
|
+
ThreadsCap,
|
|
31
32
|
)
|
|
32
33
|
from shexec.plugin import plugin_build
|
|
33
|
-
from shexec.shexec import ShExec, options_build, plugin_run
|
|
34
|
+
from shexec.shexec import ShExec, options_build, plugin_run, threads_cap
|
|
34
35
|
|
|
35
36
|
__all__: list[str] = [
|
|
36
37
|
"DirNode",
|
|
@@ -45,8 +46,10 @@ __all__: list[str] = [
|
|
|
45
46
|
"ShExecOptions",
|
|
46
47
|
"Span",
|
|
47
48
|
"StageResult",
|
|
49
|
+
"ThreadsCap",
|
|
48
50
|
"__version__",
|
|
49
51
|
"options_build",
|
|
50
52
|
"plugin_build",
|
|
51
53
|
"plugin_run",
|
|
54
|
+
"threads_cap",
|
|
52
55
|
]
|
|
@@ -7,12 +7,14 @@ Key components:
|
|
|
7
7
|
- ShExecOptions: pfdo_run's options, for the plugin
|
|
8
8
|
- PluginSpec: what a plugin built on shexec says about itself
|
|
9
9
|
- Plugin: what ``plugin_build`` returns
|
|
10
|
+
- ThreadsCap: how many jobs a run may hold at once, given the memory it has
|
|
10
11
|
- DirNode, FileJob, JobResult, JobSpec, HostFacts, RunResult, StageResult,
|
|
11
12
|
ProcessResult, TreeOptions, PfdoOptions, PfdoRunOptions: re-exported
|
|
12
13
|
- Span: pflog's record of the plugin's run, re-exported
|
|
14
|
+
- MemoryLimit: hostlib's figure for the memory the process may use, re-exported
|
|
13
15
|
|
|
14
16
|
Dependencies:
|
|
15
|
-
- pfdo_run.models, pflog.models
|
|
17
|
+
- pfdo_run.models, pflog.models, hostlib.models
|
|
16
18
|
"""
|
|
17
19
|
|
|
18
20
|
from __future__ import annotations
|
|
@@ -22,6 +24,7 @@ from collections.abc import Callable
|
|
|
22
24
|
from dataclasses import dataclass, field
|
|
23
25
|
from typing import Any
|
|
24
26
|
|
|
27
|
+
from hostlib.models import MemoryLimit
|
|
25
28
|
from pfdo_run.models import (
|
|
26
29
|
DirNode,
|
|
27
30
|
FileJob,
|
|
@@ -85,6 +88,13 @@ class PluginSpec:
|
|
|
85
88
|
``("threads", "threadsSplit")``. Each is removed from the parser, so
|
|
86
89
|
neither ChRIS's description nor the manual shows it, and its value is
|
|
87
90
|
pinned to the engine's default, or to ``defaults`` when that names it.
|
|
91
|
+
chrisHidden: engine flags offered on the command line and in the manual
|
|
92
|
+
but left out of the description ChRIS reads, by destination name. ChRIS
|
|
93
|
+
passes only the parameters its description lists, so in ChRIS each
|
|
94
|
+
keeps its default; run from a shell, the flag works.
|
|
95
|
+
jobMemory: the most memory one job needs, as ``16Gi``; empty for no figure.
|
|
96
|
+
When set, ``--threads`` is capped at the memory the process may use
|
|
97
|
+
divided by it, so that the jobs held at once fit.
|
|
88
98
|
bannerArt: the ASCII art on the manual's first page.
|
|
89
99
|
bannerTitle: the one-line title under it.
|
|
90
100
|
bannerText: the banner prose; ``{version}`` in it is replaced.
|
|
@@ -106,6 +116,8 @@ class PluginSpec:
|
|
|
106
116
|
minGpu: int = 0
|
|
107
117
|
defaults: dict[str, Any] = field(default_factory=dict)
|
|
108
118
|
omit: tuple[str, ...] = ()
|
|
119
|
+
chrisHidden: tuple[str, ...] = ()
|
|
120
|
+
jobMemory: str = ""
|
|
109
121
|
bannerArt: str = DEFAULT_BANNER_ART
|
|
110
122
|
bannerTitle: str = ""
|
|
111
123
|
bannerText: str = ""
|
|
@@ -121,7 +133,9 @@ class Plugin:
|
|
|
121
133
|
Attributes:
|
|
122
134
|
name: the plugin's name.
|
|
123
135
|
spec: what it was built from.
|
|
124
|
-
parser: its flags, defaults applied
|
|
136
|
+
parser: its flags, defaults applied: what the command line parses.
|
|
137
|
+
chrisParser: the parser ChRIS's description is made from, ``parser``
|
|
138
|
+
without the spec's ``chrisHidden`` flags.
|
|
125
139
|
main: the decorated main, the console script's target.
|
|
126
140
|
manual: its manual page.
|
|
127
141
|
|
|
@@ -133,10 +147,49 @@ class Plugin:
|
|
|
133
147
|
name: str
|
|
134
148
|
spec: PluginSpec
|
|
135
149
|
parser: argparse.ArgumentParser
|
|
150
|
+
chrisParser: argparse.ArgumentParser
|
|
136
151
|
main: Callable[..., int]
|
|
137
152
|
manual: Any
|
|
138
153
|
|
|
139
154
|
|
|
155
|
+
@dataclass
|
|
156
|
+
class ThreadsCap:
|
|
157
|
+
"""How many jobs a run may hold at once, and why.
|
|
158
|
+
|
|
159
|
+
Attributes:
|
|
160
|
+
requested: the ``--threads`` asked for; 0 is sequential.
|
|
161
|
+
allowed: what the run will use: ``requested``, or fewer when memory caps it.
|
|
162
|
+
jobBytes: the memory one job needs, in bytes; 0 when no figure was given.
|
|
163
|
+
limit: the memory the process may use, or ``None`` when it cannot be read.
|
|
164
|
+
|
|
165
|
+
Example:
|
|
166
|
+
>>> ThreadsCap(8, 3, 16 * 2**30, MemoryLimit(50 * 2**30, "cgroup")).capped()
|
|
167
|
+
True
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
requested: int
|
|
171
|
+
allowed: int
|
|
172
|
+
jobBytes: int
|
|
173
|
+
limit: MemoryLimit | None
|
|
174
|
+
|
|
175
|
+
def capped(self) -> bool:
|
|
176
|
+
"""Whether memory lowered the request."""
|
|
177
|
+
return self.allowed < self.requested
|
|
178
|
+
|
|
179
|
+
def note(self) -> str:
|
|
180
|
+
"""One line for standard error saying what was decided; empty when nothing needs saying."""
|
|
181
|
+
if self.requested <= 1 or self.jobBytes == 0:
|
|
182
|
+
return ""
|
|
183
|
+
jobGib: float = self.jobBytes / 2**30
|
|
184
|
+
if self.limit is None:
|
|
185
|
+
return f"--threads {self.requested}: memory limit unknown, not capped ({jobGib:g} GiB per job)"
|
|
186
|
+
have: str = f"{self.limit.gibibytes():.1f} GiB ({self.limit.source}) / {jobGib:g} GiB per job"
|
|
187
|
+
if self.capped():
|
|
188
|
+
fewer: str = "one at a time" if self.allowed == 0 else f"{self.allowed} at once"
|
|
189
|
+
return f"--threads {self.requested} capped to {fewer}: {have}"
|
|
190
|
+
return ""
|
|
191
|
+
|
|
192
|
+
|
|
140
193
|
__all__: list[str] = [
|
|
141
194
|
"DEFAULT_BANNER_ART",
|
|
142
195
|
"DirNode",
|
|
@@ -144,6 +197,7 @@ __all__: list[str] = [
|
|
|
144
197
|
"HostFacts",
|
|
145
198
|
"JobResult",
|
|
146
199
|
"JobSpec",
|
|
200
|
+
"MemoryLimit",
|
|
147
201
|
"PfdoOptions",
|
|
148
202
|
"PfdoRunOptions",
|
|
149
203
|
"Plugin",
|
|
@@ -153,5 +207,6 @@ __all__: list[str] = [
|
|
|
153
207
|
"ShExecOptions",
|
|
154
208
|
"Span",
|
|
155
209
|
"StageResult",
|
|
210
|
+
"ThreadsCap",
|
|
156
211
|
"TreeOptions",
|
|
157
212
|
]
|
|
@@ -18,7 +18,7 @@ Key components:
|
|
|
18
18
|
Typical usage:
|
|
19
19
|
PLUGIN: Plugin = plugin_build(PluginSpec(
|
|
20
20
|
name="freesurfer", version=__version__, title="FreeSurfer recon-all",
|
|
21
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
21
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.dcm"},
|
|
22
22
|
minMemory="8Gi", minCpu="4000m",
|
|
23
23
|
))
|
|
24
24
|
main = PLUGIN.main
|
|
@@ -30,6 +30,7 @@ Dependencies:
|
|
|
30
30
|
from __future__ import annotations
|
|
31
31
|
|
|
32
32
|
import argparse
|
|
33
|
+
import copy
|
|
33
34
|
import inspect
|
|
34
35
|
import re
|
|
35
36
|
from collections.abc import Callable, Sequence
|
|
@@ -202,6 +203,12 @@ def parser_forSpec(spec: PluginSpec) -> argparse.ArgumentParser:
|
|
|
202
203
|
def plugin_build(spec: PluginSpec, singleton: bool = True) -> Plugin:
|
|
203
204
|
"""Make a ChRIS plugin from a spec.
|
|
204
205
|
|
|
206
|
+
The command line is parsed by the spec's full parser. The description ChRIS
|
|
207
|
+
reads is made from a copy without the spec's ``chrisHidden`` flags: chris_plugin
|
|
208
|
+
describes whatever parser was registered, so the plugin registers the copy and
|
|
209
|
+
the console script parses with the original. ChRIS passes only the parameters
|
|
210
|
+
it was told of, so in ChRIS a hidden flag keeps its default.
|
|
211
|
+
|
|
205
212
|
Args:
|
|
206
213
|
spec: name, version, description for ChRIS, flag defaults, manual text.
|
|
207
214
|
singleton: register with chris_plugin's registry, which ``chris_plugin_info``
|
|
@@ -216,22 +223,31 @@ def plugin_build(spec: PluginSpec, singleton: bool = True) -> Plugin:
|
|
|
216
223
|
('x', '')
|
|
217
224
|
"""
|
|
218
225
|
parser: argparse.ArgumentParser = parser_forSpec(spec)
|
|
226
|
+
chrisParser: argparse.ArgumentParser = copy.deepcopy(parser)
|
|
227
|
+
parser_omit(chrisParser, spec.chrisHidden)
|
|
219
228
|
manual: Manual = manual_build(spec)
|
|
220
229
|
|
|
221
230
|
def body(options: argparse.Namespace, inputdir: Path, outputdir: Path) -> int:
|
|
222
|
-
return plugin_run(options, inputdir, outputdir)
|
|
231
|
+
return plugin_run(options, inputdir, outputdir, jobMemory=spec.jobMemory, name=spec.name)
|
|
223
232
|
|
|
224
233
|
body.__name__ = spec.name
|
|
225
234
|
body.__qualname__ = spec.name
|
|
226
235
|
body.__signature__ = PLUGIN_SIGNATURE # type: ignore[attr-defined] # what chris_plugin inspects; this module stringifies annotations
|
|
227
236
|
timedBody: Callable[[argparse.Namespace, Path, Path], int] = timed(spec.name)(body)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
237
|
+
|
|
238
|
+
def chris_wrap(using: argparse.ArgumentParser, register: bool) -> Callable[..., int]:
|
|
239
|
+
return chris_plugin(
|
|
240
|
+
parser=using,
|
|
241
|
+
title=spec.title,
|
|
242
|
+
category=spec.category,
|
|
243
|
+
min_memory_limit=spec.minMemory,
|
|
244
|
+
min_cpu_limit=spec.minCpu,
|
|
245
|
+
min_gpu_limit=spec.minGpu,
|
|
246
|
+
singleton=register,
|
|
247
|
+
)(timedBody)
|
|
248
|
+
|
|
249
|
+
main: Callable[..., int] = chris_wrap(parser, register=False)
|
|
250
|
+
if singleton:
|
|
251
|
+
# Registered for chris_plugin_info only; the console script is ``main``.
|
|
252
|
+
chris_wrap(chrisParser, register=True)
|
|
253
|
+
return Plugin(name=spec.name, spec=spec, parser=parser, chrisParser=chrisParser, main=main, manual=manual)
|
|
@@ -8,13 +8,14 @@ configure logging, run, report, and return an exit status.
|
|
|
8
8
|
Key components:
|
|
9
9
|
- ShExec: pfdo_run's engine, built from the plugin's arguments
|
|
10
10
|
- options_build: the plugin's arguments as ShExecOptions
|
|
11
|
+
- threads_cap: ``--threads`` against the memory the process may use
|
|
11
12
|
- plugin_run: run and report, returning the exit status
|
|
12
13
|
|
|
13
14
|
Typical usage:
|
|
14
15
|
return plugin_run(options, inputdir, outputdir)
|
|
15
16
|
|
|
16
17
|
Dependencies:
|
|
17
|
-
- pfdo_run, pftree.__main__ (logging), shexec.models
|
|
18
|
+
- pfdo_run, pftree.__main__ (logging), hostlib.facts, shexec.models
|
|
18
19
|
"""
|
|
19
20
|
|
|
20
21
|
from __future__ import annotations
|
|
@@ -24,10 +25,11 @@ import json
|
|
|
24
25
|
import sys
|
|
25
26
|
from pathlib import Path
|
|
26
27
|
|
|
28
|
+
from hostlib.facts import memory_limit, memorySize_parse
|
|
27
29
|
from pfdo_run.pfdo_run import PfdoRun
|
|
28
30
|
from pftree.__main__ import logging_configure
|
|
29
31
|
|
|
30
|
-
from shexec.models import RunResult, ShExecOptions
|
|
32
|
+
from shexec.models import MemoryLimit, RunResult, ShExecOptions, ThreadsCap
|
|
31
33
|
|
|
32
34
|
EXIT_OK: int = 0
|
|
33
35
|
EXIT_FAILED: int = 1
|
|
@@ -76,13 +78,41 @@ def result_report(options: argparse.Namespace, result: RunResult) -> None:
|
|
|
76
78
|
print(f"Elapsed time = {result.runTime:.6f} seconds")
|
|
77
79
|
|
|
78
80
|
|
|
79
|
-
def
|
|
81
|
+
def threads_cap(requested: int, jobBytes: int, limit: MemoryLimit | None) -> ThreadsCap:
|
|
82
|
+
"""How many jobs may run at once when each needs ``jobBytes`` and the process has ``limit``.
|
|
83
|
+
|
|
84
|
+
No figure for a job, or no readable limit, leaves the request as it is. Otherwise
|
|
85
|
+
the run holds at most ``limit // jobBytes`` jobs; one or none fitting is 0, the
|
|
86
|
+
engine's sequential run, which is what one job at a time means.
|
|
87
|
+
|
|
88
|
+
Example:
|
|
89
|
+
>>> threads_cap(8, 16 * 2**30, MemoryLimit(50 * 2**30, "meminfo")).allowed
|
|
90
|
+
3
|
|
91
|
+
>>> threads_cap(4, 16 * 2**30, MemoryLimit(32 * 2**30, "cgroup")).allowed
|
|
92
|
+
2
|
|
93
|
+
>>> threads_cap(4, 16 * 2**30, MemoryLimit(20 * 2**30, "cgroup")).allowed
|
|
94
|
+
0
|
|
95
|
+
"""
|
|
96
|
+
if requested <= 1 or jobBytes <= 0 or limit is None:
|
|
97
|
+
return ThreadsCap(requested=requested, allowed=requested, jobBytes=max(jobBytes, 0), limit=limit)
|
|
98
|
+
fitting: int = limit.bytes // jobBytes
|
|
99
|
+
allowed: int = requested if fitting >= requested else (fitting if fitting > 1 else 0)
|
|
100
|
+
return ThreadsCap(requested=requested, allowed=allowed, jobBytes=jobBytes, limit=limit)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def plugin_run(
|
|
104
|
+
options: argparse.Namespace, inputdir: Path, outputdir: Path, jobMemory: str = "", name: str = "shexec"
|
|
105
|
+
) -> int:
|
|
80
106
|
"""The body of the plugin's main.
|
|
81
107
|
|
|
82
108
|
Args:
|
|
83
109
|
options: the parsed flags; ``exec`` must be set.
|
|
84
110
|
inputdir: the input tree.
|
|
85
111
|
outputdir: the output tree.
|
|
112
|
+
jobMemory: the most memory one job needs, as ``16Gi``; when given,
|
|
113
|
+
``--threads`` is capped so that the jobs held at once fit in the memory
|
|
114
|
+
the process may use, and the cap is said on standard error.
|
|
115
|
+
name: the command's name, which begins its messages on standard error.
|
|
86
116
|
|
|
87
117
|
Returns:
|
|
88
118
|
0 when at least one directory's jobs all succeeded (pftree's stage rule);
|
|
@@ -90,12 +120,17 @@ def plugin_run(options: argparse.Namespace, inputdir: Path, outputdir: Path) ->
|
|
|
90
120
|
empty or a flag's value is unusable.
|
|
91
121
|
"""
|
|
92
122
|
if not options.exec:
|
|
93
|
-
print("
|
|
123
|
+
print(f"{name}: --exec is required; see {name} --man", file=sys.stderr)
|
|
94
124
|
return EXIT_USAGE
|
|
125
|
+
if jobMemory and getattr(options, "threads", 0) > 1:
|
|
126
|
+
cap: ThreadsCap = threads_cap(options.threads, memorySize_parse(jobMemory), memory_limit())
|
|
127
|
+
if cap.note():
|
|
128
|
+
print(f"{name}: {cap.note()}", file=sys.stderr)
|
|
129
|
+
options = argparse.Namespace(**{**vars(options), "threads": cap.allowed})
|
|
95
130
|
try:
|
|
96
131
|
engine: ShExec = ShExec.fromPlugin(options, inputdir, outputdir)
|
|
97
132
|
except ValueError as err:
|
|
98
|
-
print(f"
|
|
133
|
+
print(f"{name}: {err}", file=sys.stderr)
|
|
99
134
|
return EXIT_USAGE
|
|
100
135
|
logging_configure(engine.options.verbosity)
|
|
101
136
|
result: RunResult = engine.run()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shexec
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.6.0
|
|
4
4
|
Summary: Run a shell-type command over every file of an input tree: a ChRIS DS plugin on pfdo_run
|
|
5
5
|
Home-page: https://github.com/FNNDSC/pl-shexec
|
|
6
6
|
Author: FNNDSC
|
|
@@ -18,9 +18,9 @@ Requires-Python: >=3.11
|
|
|
18
18
|
Description-Content-Type: text/x-rst
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: chris_plugin>=0.4.0
|
|
21
|
-
Requires-Dist: pfdo>=6.
|
|
22
|
-
Requires-Dist: pfdo_run>=4.
|
|
23
|
-
Requires-Dist: hostlib>=1.
|
|
21
|
+
Requires-Dist: pfdo>=6.4.0
|
|
22
|
+
Requires-Dist: pfdo_run>=4.4.0
|
|
23
|
+
Requires-Dist: hostlib>=1.3.0
|
|
24
24
|
Requires-Dist: pflog>=2.0.0
|
|
25
25
|
Provides-Extra: dev
|
|
26
26
|
Requires-Dist: pytest>=8; extra == "dev"
|
|
@@ -41,13 +41,13 @@ pl-shexec
|
|
|
41
41
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
42
42
|
/incoming /outgoing
|
|
43
43
|
|
|
44
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
44
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
45
45
|
|
|
46
46
|
.. code-block:: bash
|
|
47
47
|
|
|
48
|
-
shexec --analyzeFileIndex
|
|
48
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
49
49
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
50
|
-
-s %{
|
|
50
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
51
51
|
/incoming /outgoing
|
|
52
52
|
|
|
53
53
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -57,12 +57,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
57
57
|
The template
|
|
58
58
|
------------
|
|
59
59
|
|
|
60
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
60
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
61
61
|
|
|
62
62
|
========================== ==============================================
|
|
63
63
|
``%{inputWorkingDir}`` the input directory being processed
|
|
64
64
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
65
65
|
``%{inputWorkingFile}`` the file being processed
|
|
66
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
67
|
+
name for a volume alone in it or a ``.dcm``
|
|
68
|
+
series; for a file beside other volumes, its
|
|
69
|
+
name less its extension (``scan01.nii.gz``)
|
|
66
70
|
========================== ==============================================
|
|
67
71
|
|
|
68
72
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -92,9 +96,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
92
96
|
Selecting files and directories
|
|
93
97
|
-------------------------------
|
|
94
98
|
|
|
95
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
99
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
96
100
|
|
|
97
|
-
Directories run in parallel under ``--threads
|
|
101
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
98
102
|
|
|
99
103
|
Exit status
|
|
100
104
|
-----------
|
|
@@ -108,7 +112,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
108
112
|
|
|
109
113
|
- the image carries the program, with whatever licence or data it needs;
|
|
110
114
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
111
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
115
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
112
116
|
|
|
113
117
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
114
118
|
|
|
@@ -126,7 +130,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
126
130
|
|
|
127
131
|
RECON_ALL: str = (
|
|
128
132
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
129
|
-
"-s %{
|
|
133
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
130
134
|
)
|
|
131
135
|
|
|
132
136
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -135,14 +139,14 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
135
139
|
title="FreeSurfer recon-all over every subject",
|
|
136
140
|
minMemory="8Gi",
|
|
137
141
|
minCpu="4000m",
|
|
138
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
142
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
139
143
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
140
144
|
description=DESCRIPTION,
|
|
141
145
|
examples=EXAMPLES,
|
|
142
146
|
))
|
|
143
147
|
main = PLUGIN.main
|
|
144
148
|
|
|
145
|
-
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(
|
|
149
|
+
``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. A plugin may also decline engine flags it should not offer: ``omit=(...)`` removes them from its parser, its ChRIS description and its manual, and fixes their values. A flag that suits a shell but not ChRIS goes in ``chrisHidden`` instead: it stays on the command line and in the manual and is left out of the description ChRIS reads, so in ChRIS it keeps its default. ``jobMemory="16Gi"`` says how much memory one job needs at most; ``--threads N`` is then capped at the memory the process may use, its container's or Slurm job's limit or the machine's available memory as ``hostlib.memory_limit`` reads it, divided by that figure, and the cap is said on standard error. pl-freesurfer uses both: one ``recon-all`` at a time in a 32Gi ChRIS container, several at once on a workstation or a Slurm node with the memory for them. ``plugin_build`` returns the parser with those defaults applied, the manual, and ``main`` already wrapped in pflog's ``timed`` and ``chris_plugin``, ready to be the console script. The engine, the flags, the records and the exit status are not in the spec, because they are the same for every plugin on the stack, and a fix to them reaches every plugin on its next build.
|
|
146
150
|
|
|
147
151
|
The rest of pl-freesurfer is a ``pyproject.toml`` naming ``chris_plugin`` and ``shexec`` as its dependencies, the first so that ``chris_plugin_info`` finds the plugin, a ``setup.cfg``, a Dockerfile that starts from the FreeSurfer image and installs the package, tests, and docs. pl-shexec itself is the first plugin built this way; its ``__main__`` is the same call with an empty default command. The pf* stack's ``docs/plugin.adoc`` has the template.
|
|
148
152
|
|
|
@@ -11,7 +11,9 @@ import pytest
|
|
|
11
11
|
from chris_plugin.parameters import serialize
|
|
12
12
|
|
|
13
13
|
from shexec import Plugin, PluginSpec, plugin_build
|
|
14
|
+
from shexec.models import MemoryLimit
|
|
14
15
|
from shexec.plugin import PLUGIN_SIGNATURE
|
|
16
|
+
from shexec.shexec import threads_cap
|
|
15
17
|
|
|
16
18
|
COUNT_LINES: str = "wc -l %{inputWorkingDir}/%{inputWorkingFile}"
|
|
17
19
|
|
|
@@ -132,3 +134,48 @@ def test_omittedFlags_derivedPluginStillRuns(tmp_path: Path) -> None:
|
|
|
132
134
|
plugin: Plugin = plugin_build(spec_make(omit=("threads", "threadsSplit")), singleton=False)
|
|
133
135
|
assert plugin.main(plugin.parser.parse_args(["--verbosity", "0"]), inputDir, outputDir) == 0
|
|
134
136
|
assert (outputDir / "sub-01" / "jobs.jsonl").exists()
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_chrisHidden_onCommandLineAndManual_notInDescription(capsys: pytest.CaptureFixture[str]) -> None:
|
|
140
|
+
plugin: Plugin = plugin_build(spec_make(chrisHidden=("threads", "threadsSplit")), singleton=False)
|
|
141
|
+
described: set[str] = {parameter["name"] for parameter in serialize(plugin.chrisParser)}
|
|
142
|
+
assert "threads" not in described and "threadsSplit" not in described and "verbosity" in described
|
|
143
|
+
assert "threads" in {parameter["name"] for parameter in serialize(plugin.parser)}
|
|
144
|
+
assert plugin.parser.parse_args(["--threads", "4"]).threads == 4
|
|
145
|
+
assert plugin.chrisParser.parse_args([]).threads == 0
|
|
146
|
+
with pytest.raises(SystemExit):
|
|
147
|
+
plugin.parser.parse_args(["--man"])
|
|
148
|
+
assert "[--threads" in capsys.readouterr().out
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_chrisHidden_unknownName_refused() -> None:
|
|
152
|
+
with pytest.raises(ValueError, match="thread"):
|
|
153
|
+
plugin_build(spec_make(chrisHidden=("thread",)), singleton=False)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_jobMemory_capsThreads_andSaysSo(
|
|
157
|
+
tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch
|
|
158
|
+
) -> None:
|
|
159
|
+
inputDir, outputDir = tree_make(tmp_path)
|
|
160
|
+
(inputDir / "sub-02").mkdir()
|
|
161
|
+
(inputDir / "sub-02" / "t1.nii.gz").write_text("1\n")
|
|
162
|
+
monkeypatch.setattr("shexec.shexec.memory_limit", lambda: MemoryLimit(40 * 2**30, "cgroup"))
|
|
163
|
+
plugin: Plugin = plugin_build(spec_make(jobMemory="16Gi"), singleton=False)
|
|
164
|
+
options: argparse.Namespace = plugin.parser.parse_args(["--threads", "8", "--verbosity", "0"])
|
|
165
|
+
assert plugin.main(options, inputDir, outputDir) == 0
|
|
166
|
+
err: str = capsys.readouterr().err
|
|
167
|
+
assert "linecount: --threads 8 capped to 2 at once: 40.0 GiB (cgroup) / 16 GiB per job" in err
|
|
168
|
+
assert (outputDir / "sub-01" / "jobs.jsonl").exists() and (outputDir / "sub-02" / "jobs.jsonl").exists()
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_threadsCap_note() -> None:
|
|
172
|
+
gib: int = 2**30
|
|
173
|
+
assert (
|
|
174
|
+
threads_cap(4, 16 * gib, MemoryLimit(20 * gib, "meminfo"))
|
|
175
|
+
.note()
|
|
176
|
+
.startswith("--threads 4 capped to one at a time")
|
|
177
|
+
)
|
|
178
|
+
assert threads_cap(4, 16 * gib, MemoryLimit(80 * gib, "meminfo")).note() == ""
|
|
179
|
+
assert "not capped" in threads_cap(4, 16 * gib, None).note()
|
|
180
|
+
assert threads_cap(0, 16 * gib, MemoryLimit(1, "cgroup")).allowed == 0
|
|
181
|
+
assert threads_cap(4, 0, MemoryLimit(1, "cgroup")).allowed == 4
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|