junifer 0.0.4.dev691__py3-none-any.whl → 0.0.4.dev733__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- junifer/_version.py +2 -2
- junifer/api/functions.py +11 -3
- junifer/api/queue_context/__init__.py +1 -0
- junifer/api/queue_context/gnu_parallel_local_adapter.py +258 -0
- junifer/api/queue_context/htcondor_adapter.py +4 -1
- junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py +192 -0
- junifer/api/tests/test_functions.py +2 -2
- junifer/datagrabber/dmcc13_benchmark.py +4 -1
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/METADATA +1 -1
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/RECORD +15 -13
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/AUTHORS.rst +0 -0
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/LICENSE.md +0 -0
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/WHEEL +0 -0
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/entry_points.txt +0 -0
- {junifer-0.0.4.dev691.dist-info → junifer-0.0.4.dev733.dist-info}/top_level.txt +0 -0
junifer/_version.py
CHANGED
@@ -12,5 +12,5 @@ __version__: str
|
|
12
12
|
__version_tuple__: VERSION_TUPLE
|
13
13
|
version_tuple: VERSION_TUPLE
|
14
14
|
|
15
|
-
__version__ = version = '0.0.4.
|
16
|
-
__version_tuple__ = version_tuple = (0, 0, 4, '
|
15
|
+
__version__ = version = '0.0.4.dev733'
|
16
|
+
__version_tuple__ = version_tuple = (0, 0, 4, 'dev733')
|
junifer/api/functions.py
CHANGED
@@ -18,7 +18,7 @@ from ..pipeline.registry import build
|
|
18
18
|
from ..preprocess.base import BasePreprocessor
|
19
19
|
from ..storage.base import BaseFeatureStorage
|
20
20
|
from ..utils import logger, raise_error
|
21
|
-
from .queue_context import HTCondorAdapter
|
21
|
+
from .queue_context import GnuParallelLocalAdapter, HTCondorAdapter
|
22
22
|
from .utils import yaml
|
23
23
|
|
24
24
|
|
@@ -220,7 +220,7 @@ def queue(
|
|
220
220
|
----------
|
221
221
|
config : dict
|
222
222
|
The configuration to be used for queueing the job.
|
223
|
-
kind : {"HTCondor"}
|
223
|
+
kind : {"HTCondor", "GNUParallelLocal"}
|
224
224
|
The kind of job queue system to use.
|
225
225
|
jobname : str, optional
|
226
226
|
The name of the job (default "junifer_job").
|
@@ -239,7 +239,7 @@ def queue(
|
|
239
239
|
if the ``jobdir`` exists and ``overwrite = False``.
|
240
240
|
|
241
241
|
"""
|
242
|
-
valid_kind = ["HTCondor"]
|
242
|
+
valid_kind = ["HTCondor", "GNUParallelLocal"]
|
243
243
|
if kind not in valid_kind:
|
244
244
|
raise_error(
|
245
245
|
f"Invalid value for `kind`: {kind}, "
|
@@ -311,6 +311,14 @@ def queue(
|
|
311
311
|
elements=elements,
|
312
312
|
**kwargs, # type: ignore
|
313
313
|
)
|
314
|
+
elif kind == "GNUParallelLocal":
|
315
|
+
adapter = GnuParallelLocalAdapter(
|
316
|
+
job_name=jobname,
|
317
|
+
job_dir=jobdir,
|
318
|
+
yaml_config_path=yaml_config,
|
319
|
+
elements=elements,
|
320
|
+
**kwargs, # type: ignore
|
321
|
+
)
|
314
322
|
|
315
323
|
adapter.prepare() # type: ignore
|
316
324
|
logger.info("Queue done")
|
@@ -0,0 +1,258 @@
|
|
1
|
+
"""Define concrete class for generating GNU Parallel (local) assets."""
|
2
|
+
|
3
|
+
# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
|
4
|
+
# License: AGPL
|
5
|
+
|
6
|
+
import shutil
|
7
|
+
import textwrap
|
8
|
+
from pathlib import Path
|
9
|
+
from typing import Dict, List, Optional, Tuple, Union
|
10
|
+
|
11
|
+
from ...utils import logger, make_executable, raise_error, run_ext_cmd
|
12
|
+
from .queue_context_adapter import QueueContextAdapter
|
13
|
+
|
14
|
+
|
15
|
+
__all__ = ["GnuParallelLocalAdapter"]
|
16
|
+
|
17
|
+
|
18
|
+
class GnuParallelLocalAdapter(QueueContextAdapter):
|
19
|
+
"""Class for generating commands for GNU Parallel (local).
|
20
|
+
|
21
|
+
Parameters
|
22
|
+
----------
|
23
|
+
job_name : str
|
24
|
+
The job name.
|
25
|
+
job_dir : pathlib.Path
|
26
|
+
The path to the job directory.
|
27
|
+
yaml_config_path : pathlib.Path
|
28
|
+
The path to the YAML config file.
|
29
|
+
elements : list of str or tuple
|
30
|
+
Element(s) to process. Will be used to index the DataGrabber.
|
31
|
+
pre_run : str or None, optional
|
32
|
+
Extra shell commands to source before the run (default None).
|
33
|
+
pre_collect : str or None, optional
|
34
|
+
Extra bash commands to source before the collect (default None).
|
35
|
+
env : dict, optional
|
36
|
+
The Python environment configuration. If None, will run without a
|
37
|
+
virtual environment of any kind (default None).
|
38
|
+
verbose : str, optional
|
39
|
+
The level of verbosity (default "info").
|
40
|
+
submit : bool, optional
|
41
|
+
Whether to submit the jobs (default False).
|
42
|
+
|
43
|
+
Raises
|
44
|
+
------
|
45
|
+
ValueError
|
46
|
+
If``env`` is invalid.
|
47
|
+
|
48
|
+
See Also
|
49
|
+
--------
|
50
|
+
QueueContextAdapter :
|
51
|
+
The base class for QueueContext.
|
52
|
+
HTCondorAdapter :
|
53
|
+
The concrete class for queueing via HTCondor.
|
54
|
+
|
55
|
+
"""
|
56
|
+
|
57
|
+
def __init__(
|
58
|
+
self,
|
59
|
+
job_name: str,
|
60
|
+
job_dir: Path,
|
61
|
+
yaml_config_path: Path,
|
62
|
+
elements: List[Union[str, Tuple]],
|
63
|
+
pre_run: Optional[str] = None,
|
64
|
+
pre_collect: Optional[str] = None,
|
65
|
+
env: Optional[Dict[str, str]] = None,
|
66
|
+
verbose: str = "info",
|
67
|
+
submit: bool = False,
|
68
|
+
) -> None:
|
69
|
+
"""Initialize the class."""
|
70
|
+
self._job_name = job_name
|
71
|
+
self._job_dir = job_dir
|
72
|
+
self._yaml_config_path = yaml_config_path
|
73
|
+
self._elements = elements
|
74
|
+
self._pre_run = pre_run
|
75
|
+
self._pre_collect = pre_collect
|
76
|
+
self._check_env(env)
|
77
|
+
self._verbose = verbose
|
78
|
+
self._submit = submit
|
79
|
+
|
80
|
+
self._log_dir = self._job_dir / "logs"
|
81
|
+
self._pre_run_path = self._job_dir / "pre_run.sh"
|
82
|
+
self._pre_collect_path = self._job_dir / "pre_collect.sh"
|
83
|
+
self._run_path = self._job_dir / f"run_{self._job_name}.sh"
|
84
|
+
self._collect_path = self._job_dir / f"collect_{self._job_name}.sh"
|
85
|
+
self._run_joblog_path = self._job_dir / f"run_{self._job_name}_joblog"
|
86
|
+
self._elements_file_path = self._job_dir / "elements"
|
87
|
+
|
88
|
+
def _check_env(self, env: Optional[Dict[str, str]]) -> None:
|
89
|
+
"""Check value of env parameter on init.
|
90
|
+
|
91
|
+
Parameters
|
92
|
+
----------
|
93
|
+
env : dict or None
|
94
|
+
The value of env parameter.
|
95
|
+
|
96
|
+
Raises
|
97
|
+
------
|
98
|
+
ValueError
|
99
|
+
If ``env.kind`` is invalid.
|
100
|
+
|
101
|
+
"""
|
102
|
+
# Set env related variables
|
103
|
+
if env is None:
|
104
|
+
env = {"kind": "local"}
|
105
|
+
# Check env kind
|
106
|
+
valid_env_kinds = ["conda", "venv", "local"]
|
107
|
+
if env["kind"] not in valid_env_kinds:
|
108
|
+
raise_error(
|
109
|
+
f"Invalid value for `env.kind`: {env['kind']}, "
|
110
|
+
f"must be one of {valid_env_kinds}"
|
111
|
+
)
|
112
|
+
else:
|
113
|
+
# Set variables
|
114
|
+
if env["kind"] == "local":
|
115
|
+
# No virtual environment
|
116
|
+
self._executable = "junifer"
|
117
|
+
self._arguments = ""
|
118
|
+
else:
|
119
|
+
self._executable = f"run_{env['kind']}.sh"
|
120
|
+
self._arguments = f"{env['name']} junifer"
|
121
|
+
self._exec_path = self._job_dir / self._executable
|
122
|
+
|
123
|
+
def elements(self) -> str:
|
124
|
+
"""Return elements to run."""
|
125
|
+
elements_to_run = []
|
126
|
+
for element in self._elements:
|
127
|
+
# Stringify elements if tuple for operation
|
128
|
+
str_element = (
|
129
|
+
",".join(element) if isinstance(element, tuple) else element
|
130
|
+
)
|
131
|
+
elements_to_run.append(str_element)
|
132
|
+
|
133
|
+
return "\n".join(elements_to_run)
|
134
|
+
|
135
|
+
def pre_run(self) -> str:
|
136
|
+
"""Return pre-run commands."""
|
137
|
+
fixed = (
|
138
|
+
"#!/usr/bin/env bash\n\n"
|
139
|
+
"# This script is auto-generated by junifer.\n\n"
|
140
|
+
"# Force datalad to run in non-interactive mode\n"
|
141
|
+
"DATALAD_UI_INTERACTIVE=false\n"
|
142
|
+
)
|
143
|
+
var = self._pre_run or ""
|
144
|
+
return fixed + "\n" + var
|
145
|
+
|
146
|
+
def run(self) -> str:
|
147
|
+
"""Return run commands."""
|
148
|
+
return (
|
149
|
+
"#!/usr/bin/env bash\n\n"
|
150
|
+
"# This script is auto-generated by junifer.\n\n"
|
151
|
+
"# Run pre_run.sh\n"
|
152
|
+
f"sh {self._pre_run_path.resolve()!s}\n\n"
|
153
|
+
"# Run `junifer run` using `parallel`\n"
|
154
|
+
"parallel --bar --resume --resume-failed "
|
155
|
+
f"--joblog {self._run_joblog_path} "
|
156
|
+
"--delay 60 " # wait 1 min before next job is spawned
|
157
|
+
f"--results {self._log_dir} "
|
158
|
+
f"--arg-file {self._elements_file_path.resolve()!s} "
|
159
|
+
f"{self._job_dir.resolve()!s}/{self._executable} "
|
160
|
+
f"{self._arguments} run "
|
161
|
+
f"{self._yaml_config_path.resolve()!s} "
|
162
|
+
f"--verbose {self._verbose} "
|
163
|
+
f"--element"
|
164
|
+
)
|
165
|
+
|
166
|
+
def pre_collect(self) -> str:
|
167
|
+
"""Return pre-collect commands."""
|
168
|
+
fixed = (
|
169
|
+
"#!/usr/bin/env bash\n\n"
|
170
|
+
"# This script is auto-generated by junifer.\n"
|
171
|
+
)
|
172
|
+
var = self._pre_collect or ""
|
173
|
+
return fixed + "\n" + var
|
174
|
+
|
175
|
+
def collect(self) -> str:
|
176
|
+
"""Return collect commands."""
|
177
|
+
return (
|
178
|
+
"#!/usr/bin/env bash\n\n"
|
179
|
+
"# This script is auto-generated by junifer.\n\n"
|
180
|
+
"# Run pre_collect.sh\n"
|
181
|
+
f"sh {self._pre_collect_path.resolve()!s}\n\n"
|
182
|
+
"# Run `junifer collect`\n"
|
183
|
+
f"{self._job_dir.resolve()!s}/{self._executable} "
|
184
|
+
f"{self._arguments} collect "
|
185
|
+
f"{self._yaml_config_path.resolve()!s} "
|
186
|
+
f"--verbose {self._verbose}"
|
187
|
+
)
|
188
|
+
|
189
|
+
def prepare(self) -> None:
|
190
|
+
"""Prepare assets for submission."""
|
191
|
+
logger.info("Preparing for local queue via GNU parallel")
|
192
|
+
# Copy executable if not local
|
193
|
+
if hasattr(self, "_exec_path"):
|
194
|
+
logger.info(
|
195
|
+
f"Copying {self._executable} to "
|
196
|
+
f"{self._exec_path.resolve()!s}"
|
197
|
+
)
|
198
|
+
shutil.copy(
|
199
|
+
src=Path(__file__).parent.parent / "res" / self._executable,
|
200
|
+
dst=self._exec_path,
|
201
|
+
)
|
202
|
+
make_executable(self._exec_path)
|
203
|
+
# Create elements file
|
204
|
+
logger.info(
|
205
|
+
f"Writing {self._elements_file_path.name} to "
|
206
|
+
f"{self._elements_file_path.resolve()!s}"
|
207
|
+
)
|
208
|
+
self._elements_file_path.touch()
|
209
|
+
self._elements_file_path.write_text(textwrap.dedent(self.elements()))
|
210
|
+
# Create pre run
|
211
|
+
logger.info(
|
212
|
+
f"Writing {self._pre_run_path.name} to "
|
213
|
+
f"{self._job_dir.resolve()!s}"
|
214
|
+
)
|
215
|
+
self._pre_run_path.touch()
|
216
|
+
self._pre_run_path.write_text(textwrap.dedent(self.pre_run()))
|
217
|
+
make_executable(self._pre_run_path)
|
218
|
+
# Create run
|
219
|
+
logger.info(
|
220
|
+
f"Writing {self._run_path.name} to " f"{self._job_dir.resolve()!s}"
|
221
|
+
)
|
222
|
+
self._run_path.touch()
|
223
|
+
self._run_path.write_text(textwrap.dedent(self.run()))
|
224
|
+
make_executable(self._run_path)
|
225
|
+
# Create pre collect
|
226
|
+
logger.info(
|
227
|
+
f"Writing {self._pre_collect_path.name} to "
|
228
|
+
f"{self._job_dir.resolve()!s}"
|
229
|
+
)
|
230
|
+
self._pre_collect_path.touch()
|
231
|
+
self._pre_collect_path.write_text(textwrap.dedent(self.pre_collect()))
|
232
|
+
make_executable(self._pre_collect_path)
|
233
|
+
# Create collect
|
234
|
+
logger.info(
|
235
|
+
f"Writing {self._collect_path.name} to "
|
236
|
+
f"{self._job_dir.resolve()!s}"
|
237
|
+
)
|
238
|
+
self._collect_path.touch()
|
239
|
+
self._collect_path.write_text(textwrap.dedent(self.collect()))
|
240
|
+
make_executable(self._collect_path)
|
241
|
+
# Submit if required
|
242
|
+
run_cmd = f"sh {self._run_path.resolve()!s}"
|
243
|
+
collect_cmd = f"sh {self._collect_path.resolve()!s}"
|
244
|
+
if self._submit:
|
245
|
+
logger.info(
|
246
|
+
"Shell scripts created, the following will be run:\n"
|
247
|
+
f"{run_cmd}\n"
|
248
|
+
"After successful completion of the previous step, run:\n"
|
249
|
+
f"{collect_cmd}"
|
250
|
+
)
|
251
|
+
run_ext_cmd(name=f"{self._run_path.resolve()!s}", cmd=[run_cmd])
|
252
|
+
else:
|
253
|
+
logger.info(
|
254
|
+
"Shell scripts created, to start, run:\n"
|
255
|
+
f"{run_cmd}\n"
|
256
|
+
"After successful completion of the previous step, run:\n"
|
257
|
+
f"{collect_cmd}"
|
258
|
+
)
|
@@ -66,7 +66,10 @@ class HTCondorAdapter(QueueContextAdapter):
|
|
66
66
|
|
67
67
|
See Also
|
68
68
|
--------
|
69
|
-
QueueContextAdapter :
|
69
|
+
QueueContextAdapter :
|
70
|
+
The base class for QueueContext.
|
71
|
+
GnuParallelLocalAdapter :
|
72
|
+
The concrete class for queueing via GNU Parallel (local).
|
70
73
|
|
71
74
|
"""
|
72
75
|
|
@@ -0,0 +1,192 @@
|
|
1
|
+
"""Provide tests for GnuParallelLocalAdapter."""
|
2
|
+
|
3
|
+
# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
|
4
|
+
# License: AGPL
|
5
|
+
|
6
|
+
import logging
|
7
|
+
from pathlib import Path
|
8
|
+
from typing import Dict, List, Optional, Tuple, Union
|
9
|
+
|
10
|
+
import pytest
|
11
|
+
|
12
|
+
from junifer.api.queue_context import GnuParallelLocalAdapter
|
13
|
+
|
14
|
+
|
15
|
+
def test_GnuParallelLocalAdapter_env_error() -> None:
|
16
|
+
"""Test error for invalid env kind."""
|
17
|
+
with pytest.raises(ValueError, match="Invalid value for `env.kind`"):
|
18
|
+
GnuParallelLocalAdapter(
|
19
|
+
job_name="check_env",
|
20
|
+
job_dir=Path("."),
|
21
|
+
yaml_config_path=Path("."),
|
22
|
+
elements=["sub01"],
|
23
|
+
env={"kind": "jambalaya"},
|
24
|
+
)
|
25
|
+
|
26
|
+
|
27
|
+
@pytest.mark.parametrize(
|
28
|
+
"elements, expected_text",
|
29
|
+
[
|
30
|
+
(["sub01", "sub02"], "sub01\nsub02"),
|
31
|
+
([("sub01", "ses01"), ("sub02", "ses01")], "sub01,ses01\nsub02,ses01"),
|
32
|
+
],
|
33
|
+
)
|
34
|
+
def test_GnuParallelLocalAdapter_elements(
|
35
|
+
elements: List[Union[str, Tuple]],
|
36
|
+
expected_text: str,
|
37
|
+
) -> None:
|
38
|
+
"""Test GnuParallelLocalAdapter elements().
|
39
|
+
|
40
|
+
Parameters
|
41
|
+
----------
|
42
|
+
elements : list of str or tuple
|
43
|
+
The parametrized elements.
|
44
|
+
expected_text : str
|
45
|
+
The parametrized expected text.
|
46
|
+
|
47
|
+
"""
|
48
|
+
adapter = GnuParallelLocalAdapter(
|
49
|
+
job_name="test_elements",
|
50
|
+
job_dir=Path("."),
|
51
|
+
yaml_config_path=Path("."),
|
52
|
+
elements=elements,
|
53
|
+
)
|
54
|
+
assert expected_text in adapter.elements()
|
55
|
+
|
56
|
+
|
57
|
+
@pytest.mark.parametrize(
|
58
|
+
"pre_run, expected_text",
|
59
|
+
[
|
60
|
+
(None, "# Force datalad"),
|
61
|
+
("# Check this out\n", "# Check this out"),
|
62
|
+
],
|
63
|
+
)
|
64
|
+
def test_GnuParallelLocalAdapter_pre_run(
|
65
|
+
pre_run: Optional[str], expected_text: str
|
66
|
+
) -> None:
|
67
|
+
"""Test GnuParallelLocalAdapter pre_run().
|
68
|
+
|
69
|
+
Parameters
|
70
|
+
----------
|
71
|
+
pre_run : str or None
|
72
|
+
The parametrized pre run text.
|
73
|
+
expected_text : str
|
74
|
+
The parametrized expected text.
|
75
|
+
|
76
|
+
"""
|
77
|
+
adapter = GnuParallelLocalAdapter(
|
78
|
+
job_name="test_pre_run",
|
79
|
+
job_dir=Path("."),
|
80
|
+
yaml_config_path=Path("."),
|
81
|
+
elements=["sub01"],
|
82
|
+
pre_run=pre_run,
|
83
|
+
)
|
84
|
+
assert expected_text in adapter.pre_run()
|
85
|
+
|
86
|
+
|
87
|
+
@pytest.mark.parametrize(
|
88
|
+
"pre_collect, expected_text",
|
89
|
+
[
|
90
|
+
(None, "# This script"),
|
91
|
+
("# Check this out\n", "# Check this out"),
|
92
|
+
],
|
93
|
+
)
|
94
|
+
def test_GnuParallelLocalAdapter_pre_collect(
|
95
|
+
pre_collect: Optional[str],
|
96
|
+
expected_text: str,
|
97
|
+
) -> None:
|
98
|
+
"""Test GnuParallelLocalAdapter pre_collect().
|
99
|
+
|
100
|
+
Parameters
|
101
|
+
----------
|
102
|
+
pre_collect : str or None
|
103
|
+
The parametrized pre collect text.
|
104
|
+
expected_text : str
|
105
|
+
The parametrized expected text.
|
106
|
+
|
107
|
+
"""
|
108
|
+
adapter = GnuParallelLocalAdapter(
|
109
|
+
job_name="test_pre_collect",
|
110
|
+
job_dir=Path("."),
|
111
|
+
yaml_config_path=Path("."),
|
112
|
+
elements=["sub01"],
|
113
|
+
pre_collect=pre_collect,
|
114
|
+
)
|
115
|
+
assert expected_text in adapter.pre_collect()
|
116
|
+
|
117
|
+
|
118
|
+
def test_GnuParallelLocalAdapter_run() -> None:
|
119
|
+
"""Test HTCondorAdapter run()."""
|
120
|
+
adapter = GnuParallelLocalAdapter(
|
121
|
+
job_name="test_run",
|
122
|
+
job_dir=Path("."),
|
123
|
+
yaml_config_path=Path("."),
|
124
|
+
elements=["sub01"],
|
125
|
+
)
|
126
|
+
assert "run" in adapter.run()
|
127
|
+
|
128
|
+
|
129
|
+
def test_GnuParallelLocalAdapter_collect() -> None:
|
130
|
+
"""Test HTCondorAdapter collect()."""
|
131
|
+
adapter = GnuParallelLocalAdapter(
|
132
|
+
job_name="test_run_collect",
|
133
|
+
job_dir=Path("."),
|
134
|
+
yaml_config_path=Path("."),
|
135
|
+
elements=["sub01"],
|
136
|
+
)
|
137
|
+
assert "collect" in adapter.collect()
|
138
|
+
|
139
|
+
|
140
|
+
@pytest.mark.parametrize(
|
141
|
+
"env",
|
142
|
+
[
|
143
|
+
{"kind": "conda", "name": "junifer"},
|
144
|
+
{"kind": "venv", "name": "./junifer"},
|
145
|
+
],
|
146
|
+
)
|
147
|
+
def test_GnuParallelLocalAdapter_prepare(
|
148
|
+
tmp_path: Path,
|
149
|
+
monkeypatch: pytest.MonkeyPatch,
|
150
|
+
caplog: pytest.LogCaptureFixture,
|
151
|
+
env: Dict[str, str],
|
152
|
+
) -> None:
|
153
|
+
"""Test GnuParallelLocalAdapter prepare().
|
154
|
+
|
155
|
+
Parameters
|
156
|
+
----------
|
157
|
+
tmp_path : pathlib.Path
|
158
|
+
The path to the test directory.
|
159
|
+
monkeypatch : pytest.MonkeyPatch
|
160
|
+
The pytest.MonkeyPatch object.
|
161
|
+
caplog : pytest.LogCaptureFixture
|
162
|
+
The pytest.LogCaptureFixture object.
|
163
|
+
env : dict
|
164
|
+
The parametrized Python environment config.
|
165
|
+
|
166
|
+
"""
|
167
|
+
with monkeypatch.context() as m:
|
168
|
+
m.chdir(tmp_path)
|
169
|
+
with caplog.at_level(logging.DEBUG):
|
170
|
+
adapter = GnuParallelLocalAdapter(
|
171
|
+
job_name="test_prepare",
|
172
|
+
job_dir=tmp_path,
|
173
|
+
yaml_config_path=tmp_path / "config.yaml",
|
174
|
+
elements=["sub01"],
|
175
|
+
env=env,
|
176
|
+
)
|
177
|
+
adapter.prepare()
|
178
|
+
|
179
|
+
assert "GNU parallel" in caplog.text
|
180
|
+
assert f"Copying run_{env['kind']}" in caplog.text
|
181
|
+
assert "Writing pre_run.sh" in caplog.text
|
182
|
+
assert "Writing run_test_prepare.sh" in caplog.text
|
183
|
+
assert "Writing pre_collect.sh" in caplog.text
|
184
|
+
assert "Writing collect_test_prepare.sh" in caplog.text
|
185
|
+
assert "Shell scripts created" in caplog.text
|
186
|
+
|
187
|
+
assert adapter._exec_path.stat().st_size != 0
|
188
|
+
assert adapter._elements_file_path.stat().st_size != 0
|
189
|
+
assert adapter._pre_run_path.stat().st_size != 0
|
190
|
+
assert adapter._run_path.stat().st_size != 0
|
191
|
+
assert adapter._pre_collect_path.stat().st_size != 0
|
192
|
+
assert adapter._collect_path.stat().st_size != 0
|
@@ -570,14 +570,14 @@ def test_reset_queue(
|
|
570
570
|
},
|
571
571
|
"mem": "8G",
|
572
572
|
},
|
573
|
-
kind="
|
573
|
+
kind="GNUParallelLocal",
|
574
574
|
jobname=job_name,
|
575
575
|
)
|
576
576
|
# Reset operation
|
577
577
|
reset(
|
578
578
|
config={
|
579
579
|
"storage": storage,
|
580
|
-
"queue": {"jobname": job_name},
|
580
|
+
"queue": {"kind": "GNUParallelLocal", "jobname": job_name},
|
581
581
|
}
|
582
582
|
)
|
583
583
|
|
@@ -293,7 +293,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
|
|
293
293
|
"f5004cr",
|
294
294
|
"f5407sl",
|
295
295
|
"f5416zj",
|
296
|
-
"
|
296
|
+
"f8113do",
|
297
297
|
"f8570ui",
|
298
298
|
"f9057kp",
|
299
299
|
]
|
@@ -324,6 +324,9 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
|
|
324
324
|
run = "1"
|
325
325
|
else:
|
326
326
|
run = "2"
|
327
|
+
# Bypass for f5416zj for not having wave1rea session
|
328
|
+
if subject == "f5416zj" and session == "wave1rea":
|
329
|
+
continue
|
327
330
|
elems.append((subject, session, task, phase_encoding, run))
|
328
331
|
|
329
332
|
return elems
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: junifer
|
3
|
-
Version: 0.0.4.
|
3
|
+
Version: 0.0.4.dev733
|
4
4
|
Summary: JUelich NeuroImaging FEature extractoR
|
5
5
|
Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
|
6
6
|
Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
|
@@ -1,15 +1,17 @@
|
|
1
1
|
junifer/__init__.py,sha256=x1UR2jUcrUdm2HNl-3Qvyi4UUrU6ms5qm2qcmNY7zZk,391
|
2
|
-
junifer/_version.py,sha256=
|
2
|
+
junifer/_version.py,sha256=tIgEwo3Tru6RrqPrJRNhyn8fYa0WPQ7N2K1bhKqn7YE,428
|
3
3
|
junifer/stats.py,sha256=sU5IZ2qFZWbzgSutQS_z42miIVItpSGmQYBn6KkD5fA,6162
|
4
4
|
junifer/api/__init__.py,sha256=YILu9M7SC0Ri4CVd90fELH2OnK_gvCYAXCoqBNCFE8E,257
|
5
5
|
junifer/api/cli.py,sha256=auw38tIH7ipTnaADM7on0or7zauY-BFII8nd-eRUEJs,13664
|
6
6
|
junifer/api/decorators.py,sha256=8bnwHPAe7VgzKxl--M_e0umdAlTVSzaJQHEJZ5kof5k,2580
|
7
|
-
junifer/api/functions.py,sha256=
|
7
|
+
junifer/api/functions.py,sha256=bH8tkbkpZmU8jiQqvSsbIRp-B__Uie0BNZy_yVwRzs8,11579
|
8
8
|
junifer/api/parser.py,sha256=a3SSC2xO-PF1pqXZXFq8Sh9aVd-dmHolJbCkGyOUTAM,4416
|
9
9
|
junifer/api/utils.py,sha256=dyjTdPMwX9qeCrn8SQT2Pjshfnu-y1FEyujV7lCzvm0,3333
|
10
|
-
junifer/api/queue_context/__init__.py,sha256=
|
11
|
-
junifer/api/queue_context/
|
10
|
+
junifer/api/queue_context/__init__.py,sha256=x0fT0ax-jPI0Fefg2quJ6VCIwhJ9rUQEjCNqxDXw7WM,287
|
11
|
+
junifer/api/queue_context/gnu_parallel_local_adapter.py,sha256=53zwa95agzx3hM1PMmHlOaApZ-SQpbQjVuac9vbpTLk,9113
|
12
|
+
junifer/api/queue_context/htcondor_adapter.py,sha256=rsVwXP1crkopdwIKVU369crX7WCTr63iE8WpQEX8e9M,12785
|
12
13
|
junifer/api/queue_context/queue_context_adapter.py,sha256=a6UE8xavDfuaZbkWYsayVs6l-rwIrbpFSpqSyHsEeYY,1577
|
14
|
+
junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py,sha256=HD_fD-nNm77EVjNpcZ968fqYUWNzylkHvFH7sFEFXN4,5393
|
13
15
|
junifer/api/queue_context/tests/test_htcondor_adapter.py,sha256=XmtdqoUmYTdkwGqRvIX52_n-vaVnPfvz81K4iSc6P0c,7255
|
14
16
|
junifer/api/res/run_conda.sh,sha256=eskIn-fotITOlh--IUQKXh-SBOZkDkK8QvDcdQsyJ0M,509
|
15
17
|
junifer/api/res/run_venv.sh,sha256=OF-LOdOdpjCU0JX6Dz1boG4ErPutTiVVD_N7j520Fvk,499
|
@@ -29,7 +31,7 @@ junifer/api/res/fsl/run_fsl_docker.sh,sha256=mRLtZo0OgDwleoee2MG6rYI37HVuGNk9zOA
|
|
29
31
|
junifer/api/res/fsl/std2imgcoord,sha256=-X5wRH6XMl0yqnTACJX6MFhO8DFOEWg42MHRxGvimXg,49
|
30
32
|
junifer/api/tests/test_api_utils.py,sha256=zDRQiqFOaoO02FpzJCxEbZvsP4u4__Yr25e5k5MJwuI,2713
|
31
33
|
junifer/api/tests/test_cli.py,sha256=hou8Jn31v0SfjAwf5zIqQEbA1nxvL6uvVZUjnghAG3k,8553
|
32
|
-
junifer/api/tests/test_functions.py,sha256=
|
34
|
+
junifer/api/tests/test_functions.py,sha256=VVciSskfgFdQMK2s8qh-5sURrGUz-wndRPepuTEfZOM,15567
|
33
35
|
junifer/api/tests/test_parser.py,sha256=eUz2JPVb0_cxvoeI1O_C5PMNs5v_lDzGsN6fV1VW5Eg,6109
|
34
36
|
junifer/api/tests/data/gmd_mean.yaml,sha256=Ohb_C5cfQMK-59U9O1ZhejXyBtzLc5Y4cv8QyYq2azg,330
|
35
37
|
junifer/api/tests/data/gmd_mean_htcondor.yaml,sha256=f7NLv_KIJXTiPNFmOWl2Vw8EfwojhfkGtwbh5prbd6w,417
|
@@ -82,7 +84,7 @@ junifer/data/tests/test_template_spaces.py,sha256=EDsGYBAwjqjOANDNssXaaXPkNv9P65
|
|
82
84
|
junifer/datagrabber/__init__.py,sha256=pZHJIY8nAlbVngsyRScE6a6GKbytiwjJB7SdJNqIbl4,680
|
83
85
|
junifer/datagrabber/base.py,sha256=clErdDX5jz8LTjwPTQPCv3jfELCJ2vObg7ks8imAx8Y,6224
|
84
86
|
junifer/datagrabber/datalad_base.py,sha256=dDaBiIePPP6-G4ycgBMxTcXxs4vkg-yDS3OBURK4VGs,10731
|
85
|
-
junifer/datagrabber/dmcc13_benchmark.py,sha256=
|
87
|
+
junifer/datagrabber/dmcc13_benchmark.py,sha256=lZpGd_6mUmFPcaqERVWPHIzJq_h_muvh1ifwxj6qqdA,11387
|
86
88
|
junifer/datagrabber/multiple.py,sha256=eXQIsvSNvD8GuEITjMaMoi1GwoeyWXXbQMRi-f2qgc4,4923
|
87
89
|
junifer/datagrabber/pattern.py,sha256=uGobDTrRhq-XPNIz0z3eIeIzmNOYFajEXwa7OWqno5g,10720
|
88
90
|
junifer/datagrabber/pattern_datalad.py,sha256=SyXhM0XQEF45poNaWyVAG054BSCYyqJ-JUwbRo05nyk,1653
|
@@ -245,10 +247,10 @@ junifer/utils/logging.py,sha256=furcU3XIUpUvnpe4PEwzWWIWgmH4j2ZA4MQdvSGWjj0,9216
|
|
245
247
|
junifer/utils/tests/test_fs.py,sha256=WQS7cKlKEZ742CIuiOYYpueeAhY9PqlastfDVpVVtvE,923
|
246
248
|
junifer/utils/tests/test_helpers.py,sha256=k5qqfxK8dFyuewTJyR1Qn6-nFaYNuVr0ysc18bfPjyU,929
|
247
249
|
junifer/utils/tests/test_logging.py,sha256=l8oo-AiBV7H6_IzlsNcj__cLeZBUvgIGoaMszD9VaJg,7754
|
248
|
-
junifer-0.0.4.
|
249
|
-
junifer-0.0.4.
|
250
|
-
junifer-0.0.4.
|
251
|
-
junifer-0.0.4.
|
252
|
-
junifer-0.0.4.
|
253
|
-
junifer-0.0.4.
|
254
|
-
junifer-0.0.4.
|
250
|
+
junifer-0.0.4.dev733.dist-info/AUTHORS.rst,sha256=rmULKpchpSol4ExWFdm-qu4fkpSZPYqIESVJBZtGb6E,163
|
251
|
+
junifer-0.0.4.dev733.dist-info/LICENSE.md,sha256=MqCnOBu8uXsEOzRZWh9EBVfVz-kE9NkXcLCrtGXo2yU,34354
|
252
|
+
junifer-0.0.4.dev733.dist-info/METADATA,sha256=k3yrPJ0o7e_weVABaPHO7Q_cl65HeeXMizcqiEEepiA,8235
|
253
|
+
junifer-0.0.4.dev733.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
254
|
+
junifer-0.0.4.dev733.dist-info/entry_points.txt,sha256=DxFvKq0pOqRunAK0FxwJcoDfV1-dZvsFDpD5HRqSDhw,48
|
255
|
+
junifer-0.0.4.dev733.dist-info/top_level.txt,sha256=4bAq1R2QFQ4b3hohjys2JBvxrl0GKk5LNFzYvz9VGcA,8
|
256
|
+
junifer-0.0.4.dev733.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|