shexec 2.5.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shexec-2.5.0/shexec.egg-info → shexec-2.7.0}/PKG-INFO +16 -12
- {shexec-2.5.0 → shexec-2.7.0}/README.rst +13 -9
- {shexec-2.5.0 → shexec-2.7.0}/pyproject.toml +1 -1
- {shexec-2.5.0 → shexec-2.7.0}/shexec/__init__.py +1 -1
- {shexec-2.5.0 → shexec-2.7.0}/shexec/models.py +4 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec/plugin.py +2 -1
- {shexec-2.5.0 → shexec-2.7.0/shexec.egg-info}/PKG-INFO +16 -12
- {shexec-2.5.0 → shexec-2.7.0}/shexec.egg-info/requires.txt +2 -2
- {shexec-2.5.0 → shexec-2.7.0}/tests/test_plugin.py +15 -0
- {shexec-2.5.0 → shexec-2.7.0}/LICENSE +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/setup.cfg +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec/__main__.py +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec/shexec.py +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec.egg-info/SOURCES.txt +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec.egg-info/dependency_links.txt +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec.egg-info/entry_points.txt +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/shexec.egg-info/top_level.txt +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/tests/test_cli.py +0 -0
- {shexec-2.5.0 → shexec-2.7.0}/tests/test_shexec.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shexec
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Run a shell-type command over every file of an input tree: a ChRIS DS plugin on pfdo_run
|
|
5
5
|
Home-page: https://github.com/FNNDSC/pl-shexec
|
|
6
6
|
Author: FNNDSC
|
|
@@ -18,8 +18,8 @@ Requires-Python: >=3.11
|
|
|
18
18
|
Description-Content-Type: text/x-rst
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: chris_plugin>=0.4.0
|
|
21
|
-
Requires-Dist: pfdo>=6.
|
|
22
|
-
Requires-Dist: pfdo_run>=4.
|
|
21
|
+
Requires-Dist: pfdo>=6.4.0
|
|
22
|
+
Requires-Dist: pfdo_run>=4.4.0
|
|
23
23
|
Requires-Dist: hostlib>=1.3.0
|
|
24
24
|
Requires-Dist: pflog>=2.0.0
|
|
25
25
|
Provides-Extra: dev
|
|
@@ -41,13 +41,13 @@ pl-shexec
|
|
|
41
41
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
42
42
|
/incoming /outgoing
|
|
43
43
|
|
|
44
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
44
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
45
45
|
|
|
46
46
|
.. code-block:: bash
|
|
47
47
|
|
|
48
|
-
shexec --analyzeFileIndex
|
|
48
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
49
49
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
50
|
-
-s %{
|
|
50
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
51
51
|
/incoming /outgoing
|
|
52
52
|
|
|
53
53
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -57,12 +57,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
57
57
|
The template
|
|
58
58
|
------------
|
|
59
59
|
|
|
60
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
60
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
61
61
|
|
|
62
62
|
========================== ==============================================
|
|
63
63
|
``%{inputWorkingDir}`` the input directory being processed
|
|
64
64
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
65
65
|
``%{inputWorkingFile}`` the file being processed
|
|
66
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
67
|
+
name for a volume alone in it or a ``.dcm``
|
|
68
|
+
series; for a file beside other volumes, its
|
|
69
|
+
name less its extension (``scan01.nii.gz``)
|
|
66
70
|
========================== ==============================================
|
|
67
71
|
|
|
68
72
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -92,9 +96,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
92
96
|
Selecting files and directories
|
|
93
97
|
-------------------------------
|
|
94
98
|
|
|
95
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
99
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
96
100
|
|
|
97
|
-
Directories run in parallel under ``--threads
|
|
101
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
98
102
|
|
|
99
103
|
Exit status
|
|
100
104
|
-----------
|
|
@@ -108,7 +112,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
108
112
|
|
|
109
113
|
- the image carries the program, with whatever licence or data it needs;
|
|
110
114
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
111
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
115
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
112
116
|
|
|
113
117
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
114
118
|
|
|
@@ -126,7 +130,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
126
130
|
|
|
127
131
|
RECON_ALL: str = (
|
|
128
132
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
129
|
-
"-s %{
|
|
133
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
130
134
|
)
|
|
131
135
|
|
|
132
136
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -135,7 +139,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
135
139
|
title="FreeSurfer recon-all over every subject",
|
|
136
140
|
minMemory="8Gi",
|
|
137
141
|
minCpu="4000m",
|
|
138
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
142
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
139
143
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
140
144
|
description=DESCRIPTION,
|
|
141
145
|
examples=EXAMPLES,
|
|
@@ -12,13 +12,13 @@ pl-shexec
|
|
|
12
12
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
13
13
|
/incoming /outgoing
|
|
14
14
|
|
|
15
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
15
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
16
16
|
|
|
17
17
|
.. code-block:: bash
|
|
18
18
|
|
|
19
|
-
shexec --analyzeFileIndex
|
|
19
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
20
20
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
21
|
-
-s %{
|
|
21
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
22
22
|
/incoming /outgoing
|
|
23
23
|
|
|
24
24
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -28,12 +28,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
28
28
|
The template
|
|
29
29
|
------------
|
|
30
30
|
|
|
31
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
31
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
32
32
|
|
|
33
33
|
========================== ==============================================
|
|
34
34
|
``%{inputWorkingDir}`` the input directory being processed
|
|
35
35
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
36
36
|
``%{inputWorkingFile}`` the file being processed
|
|
37
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
38
|
+
name for a volume alone in it or a ``.dcm``
|
|
39
|
+
series; for a file beside other volumes, its
|
|
40
|
+
name less its extension (``scan01.nii.gz``)
|
|
37
41
|
========================== ==============================================
|
|
38
42
|
|
|
39
43
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -63,9 +67,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
63
67
|
Selecting files and directories
|
|
64
68
|
-------------------------------
|
|
65
69
|
|
|
66
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
70
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
67
71
|
|
|
68
|
-
Directories run in parallel under ``--threads
|
|
72
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
69
73
|
|
|
70
74
|
Exit status
|
|
71
75
|
-----------
|
|
@@ -79,7 +83,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
79
83
|
|
|
80
84
|
- the image carries the program, with whatever licence or data it needs;
|
|
81
85
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
82
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
86
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
83
87
|
|
|
84
88
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
85
89
|
|
|
@@ -97,7 +101,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
97
101
|
|
|
98
102
|
RECON_ALL: str = (
|
|
99
103
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
100
|
-
"-s %{
|
|
104
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
101
105
|
)
|
|
102
106
|
|
|
103
107
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -106,7 +110,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
106
110
|
title="FreeSurfer recon-all over every subject",
|
|
107
111
|
minMemory="8Gi",
|
|
108
112
|
minCpu="4000m",
|
|
109
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
113
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
110
114
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
111
115
|
description=DESCRIPTION,
|
|
112
116
|
examples=EXAMPLES,
|
|
@@ -13,7 +13,7 @@ authors = [{ email = "dev@babyMRI.org" }]
|
|
|
13
13
|
# fields, which the [project] table does not emit; setup.cfg supplies them, and the
|
|
14
14
|
# author name lives there alone so Author-email stays a bare address.
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
|
-
dependencies = ["chris_plugin>=0.4.0", "pfdo>=6.
|
|
16
|
+
dependencies = ["chris_plugin>=0.4.0", "pfdo>=6.4.0", "pfdo_run>=4.4.0", "hostlib>=1.3.0", "pflog>=2.0.0"]
|
|
17
17
|
classifiers = [
|
|
18
18
|
"Programming Language :: Python :: 3",
|
|
19
19
|
"Programming Language :: Python :: 3.11",
|
|
@@ -83,6 +83,9 @@ class PluginSpec:
|
|
|
83
83
|
minMemory: the memory ChRIS reserves, as ``2Gi``.
|
|
84
84
|
minCpu: the CPU ChRIS reserves, in millicores, as ``1000m``.
|
|
85
85
|
minGpu: GPUs required; 0 for none.
|
|
86
|
+
maxGpu: GPUs a run may be given; 0 means as many as ``minGpu``. CUBE
|
|
87
|
+
refuses a description whose minimum exceeds its maximum, and
|
|
88
|
+
chris_plugin writes a maximum of 0 unless told otherwise.
|
|
86
89
|
defaults: flag defaults, ``exec`` above all; a derived plugin's usual command.
|
|
87
90
|
omit: engine flags this plugin does not offer, by destination name, as
|
|
88
91
|
``("threads", "threadsSplit")``. Each is removed from the parser, so
|
|
@@ -114,6 +117,7 @@ class PluginSpec:
|
|
|
114
117
|
minMemory: str = "2Gi"
|
|
115
118
|
minCpu: str = "1000m"
|
|
116
119
|
minGpu: int = 0
|
|
120
|
+
maxGpu: int = 0
|
|
117
121
|
defaults: dict[str, Any] = field(default_factory=dict)
|
|
118
122
|
omit: tuple[str, ...] = ()
|
|
119
123
|
chrisHidden: tuple[str, ...] = ()
|
|
@@ -18,7 +18,7 @@ Key components:
|
|
|
18
18
|
Typical usage:
|
|
19
19
|
PLUGIN: Plugin = plugin_build(PluginSpec(
|
|
20
20
|
name="freesurfer", version=__version__, title="FreeSurfer recon-all",
|
|
21
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
21
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.dcm"},
|
|
22
22
|
minMemory="8Gi", minCpu="4000m",
|
|
23
23
|
))
|
|
24
24
|
main = PLUGIN.main
|
|
@@ -243,6 +243,7 @@ def plugin_build(spec: PluginSpec, singleton: bool = True) -> Plugin:
|
|
|
243
243
|
min_memory_limit=spec.minMemory,
|
|
244
244
|
min_cpu_limit=spec.minCpu,
|
|
245
245
|
min_gpu_limit=spec.minGpu,
|
|
246
|
+
max_gpu_limit=max(spec.maxGpu, spec.minGpu),
|
|
246
247
|
singleton=register,
|
|
247
248
|
)(timedBody)
|
|
248
249
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: shexec
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: Run a shell-type command over every file of an input tree: a ChRIS DS plugin on pfdo_run
|
|
5
5
|
Home-page: https://github.com/FNNDSC/pl-shexec
|
|
6
6
|
Author: FNNDSC
|
|
@@ -18,8 +18,8 @@ Requires-Python: >=3.11
|
|
|
18
18
|
Description-Content-Type: text/x-rst
|
|
19
19
|
License-File: LICENSE
|
|
20
20
|
Requires-Dist: chris_plugin>=0.4.0
|
|
21
|
-
Requires-Dist: pfdo>=6.
|
|
22
|
-
Requires-Dist: pfdo_run>=4.
|
|
21
|
+
Requires-Dist: pfdo>=6.4.0
|
|
22
|
+
Requires-Dist: pfdo_run>=4.4.0
|
|
23
23
|
Requires-Dist: hostlib>=1.3.0
|
|
24
24
|
Requires-Dist: pflog>=2.0.0
|
|
25
25
|
Provides-Extra: dev
|
|
@@ -41,13 +41,13 @@ pl-shexec
|
|
|
41
41
|
%{outputWorkingDir}/%{inputWorkingFile|rmext}.png" \
|
|
42
42
|
/incoming /outgoing
|
|
43
43
|
|
|
44
|
-
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per
|
|
44
|
+
It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per scan, each NIfTI file or DICOM series, is:
|
|
45
45
|
|
|
46
46
|
.. code-block:: bash
|
|
47
47
|
|
|
48
|
-
shexec --analyzeFileIndex
|
|
48
|
+
shexec --analyzeFileIndex v --fileFilter 'glob:*.nii.gz,glob:*.dcm' \
|
|
49
49
|
--exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
|
|
50
|
-
-s %{
|
|
50
|
+
-s %{volumeName} -sd %{outputWorkingDir}" \
|
|
51
51
|
/incoming /outgoing
|
|
52
52
|
|
|
53
53
|
pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
|
|
@@ -57,12 +57,16 @@ The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pf
|
|
|
57
57
|
The template
|
|
58
58
|
------------
|
|
59
59
|
|
|
60
|
-
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written.
|
|
60
|
+
``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Four tags describe the job:
|
|
61
61
|
|
|
62
62
|
========================== ==============================================
|
|
63
63
|
``%{inputWorkingDir}`` the input directory being processed
|
|
64
64
|
``%{outputWorkingDir}`` its counterpart in the output tree
|
|
65
65
|
``%{inputWorkingFile}`` the file being processed
|
|
66
|
+
``%{volumeName}`` the volume the file stands for: the directory's
|
|
67
|
+
name for a volume alone in it or a ``.dcm``
|
|
68
|
+
series; for a file beside other volumes, its
|
|
69
|
+
name less its extension (``scan01.nii.gz``)
|
|
66
70
|
========================== ==============================================
|
|
67
71
|
|
|
68
72
|
Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
|
|
@@ -92,9 +96,9 @@ The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one
|
|
|
92
96
|
Selecting files and directories
|
|
93
97
|
-------------------------------
|
|
94
98
|
|
|
95
|
-
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
|
|
99
|
+
The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, ``v`` one per volume, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series. ``v`` is one command per volume: the ``.dcm`` files of a directory together are one, started on the first, and every other selected file is one of its own, so a folder of three NIfTI scans gives three commands and a series beside them a fourth. ``--fileFilter`` decides which files count, so a filter naming the input types keeps sidecars such as ``.json`` out. Any other choice that leaves out a volume ``v`` would have run is named in a warning.
|
|
96
100
|
|
|
97
|
-
Directories run in parallel under ``--threads
|
|
101
|
+
Directories run in parallel under ``--threads``; the volumes of one directory run one after another. At ``--verbosity 2`` each command's output streams to the console as it runs.
|
|
98
102
|
|
|
99
103
|
Exit status
|
|
100
104
|
-----------
|
|
@@ -108,7 +112,7 @@ A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK,
|
|
|
108
112
|
|
|
109
113
|
- the image carries the program, with whatever licence or data it needs;
|
|
110
114
|
- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
|
|
111
|
-
- the default ``--analyzeFileIndex`` chooses the unit, ``
|
|
115
|
+
- the default ``--analyzeFileIndex`` chooses the unit, ``v`` for one run per volume, a scan file or a DICOM series, ``f`` for one run per directory, ``a`` for one run per file.
|
|
112
116
|
|
|
113
117
|
The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
|
|
114
118
|
|
|
@@ -126,7 +130,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
126
130
|
|
|
127
131
|
RECON_ALL: str = (
|
|
128
132
|
"recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
|
|
129
|
-
"-s %{
|
|
133
|
+
"-s %{volumeName} -sd %{outputWorkingDir}"
|
|
130
134
|
)
|
|
131
135
|
|
|
132
136
|
PLUGIN = plugin_build(PluginSpec(
|
|
@@ -135,7 +139,7 @@ pl-shexec is a library as well as a plugin, in the same way pfdo_run is a librar
|
|
|
135
139
|
title="FreeSurfer recon-all over every subject",
|
|
136
140
|
minMemory="8Gi",
|
|
137
141
|
minCpu="4000m",
|
|
138
|
-
defaults={"exec": RECON_ALL, "analyzeFileIndex": "
|
|
142
|
+
defaults={"exec": RECON_ALL, "analyzeFileIndex": "v", "fileFilter": "glob:*.nii.gz,glob:*.nii,glob:*.mgz,glob:*.dcm"},
|
|
139
143
|
bannerTitle="FreeSurfer recon-all over an input tree",
|
|
140
144
|
description=DESCRIPTION,
|
|
141
145
|
examples=EXAMPLES,
|
|
@@ -8,6 +8,7 @@ import json
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
|
|
10
10
|
import pytest
|
|
11
|
+
from chris_plugin import _registration as chris_registration
|
|
11
12
|
from chris_plugin.parameters import serialize
|
|
12
13
|
|
|
13
14
|
from shexec import Plugin, PluginSpec, plugin_build
|
|
@@ -179,3 +180,17 @@ def test_threadsCap_note() -> None:
|
|
|
179
180
|
assert "not capped" in threads_cap(4, 16 * gib, None).note()
|
|
180
181
|
assert threads_cap(0, 16 * gib, MemoryLimit(1, "cgroup")).allowed == 0
|
|
181
182
|
assert threads_cap(4, 0, MemoryLimit(1, "cgroup")).allowed == 4
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def test_minGpu_describesAMaximumCubeAccepts(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
186
|
+
monkeypatch.setattr(chris_registration, "_memory", [])
|
|
187
|
+
plugin_build(spec_make(minGpu=1), singleton=True)
|
|
188
|
+
details = chris_registration.get_registered()
|
|
189
|
+
assert (details.min_gpu_limit, details.max_gpu_limit) == (1, 1)
|
|
190
|
+
monkeypatch.setattr(chris_registration, "_memory", [])
|
|
191
|
+
plugin_build(spec_make(minGpu=1, maxGpu=2), singleton=True)
|
|
192
|
+
assert chris_registration.get_registered().max_gpu_limit == 2
|
|
193
|
+
monkeypatch.setattr(chris_registration, "_memory", [])
|
|
194
|
+
plugin_build(spec_make(), singleton=True)
|
|
195
|
+
details = chris_registration.get_registered()
|
|
196
|
+
assert (details.min_gpu_limit, details.max_gpu_limit) == (0, 0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|