junifer 0.0.3.dev188__py3-none-any.whl → 0.0.4__py3-none-any.whl

junifer/api/functions.py

@@ -6,8 +6,6 @@
 # License: AGPL
 
 import shutil
-import subprocess
-import textwrap
 import typing
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
@@ -15,11 +13,12 @@ from typing import Dict, List, Optional, Tuple, Union
 from ..datagrabber.base import BaseDataGrabber
 from ..markers.base import BaseMarker
 from ..markers.collection import MarkerCollection
+from ..pipeline import WorkDirManager
 from ..pipeline.registry import build
 from ..preprocess.base import BasePreprocessor
 from ..storage.base import BaseFeatureStorage
 from ..utils import logger, raise_error
-from ..utils.fs import make_executable
+from .queue_context import GnuParallelLocalAdapter, HTCondorAdapter
 from .utils import yaml
 
 
@@ -80,7 +79,7 @@ def run(
     datagrabber: Dict,
     markers: List[Dict],
     storage: Dict,
-    preprocessor: Optional[Dict] = None,
+    preprocessors: Optional[List[Dict]] = None,
     elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None,
 ) -> None:
     """Run the pipeline on the selected element.
@@ -103,10 +102,10 @@ def run(
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
         init function.
-    preprocessor : dict, optional
-        Preprocessor to use. Must have a key ``kind`` with the kind of
-        preprocessor to use. All other keys are passed to the preprocessor
-        init function (default None).
+    preprocessors : list of dict, optional
+        List of preprocessors to use. Each preprocessor is a dict with at
+        least a key ``kind`` specifying the preprocessor to use. All other keys
+        are passed to the preprocessor init function (default None).
     elements : str or tuple or list of str or tuple, optional
         Element(s) to process. Will be used to index the DataGrabber
         (default None).
@@ -115,6 +114,8 @@ def run(
     # Convert str to Path
     if isinstance(workdir, str):
         workdir = Path(workdir)
+    # Initiate working directory manager
+    WorkDirManager(workdir)
 
     if not isinstance(elements, list) and elements is not None:
         elements = [elements]
@@ -149,21 +150,29 @@ def run(
     storage_object = typing.cast(BaseFeatureStorage, storage_object)
 
     # Get preprocessor to use (if provided)
-    if preprocessor is not None:
-        preprocessor_object = _get_preprocessor(preprocessor)
+    if preprocessors is not None:
+        _preprocessors = [x.copy() for x in preprocessors]
+        built_preprocessors = []
+        for preprocessor in _preprocessors:
+            preprocessor_object = _get_preprocessor(preprocessor)
+            built_preprocessors.append(preprocessor_object)
     else:
-        preprocessor_object = None
+        built_preprocessors = None
 
     # Create new marker collection
     mc = MarkerCollection(
         markers=built_markers,
-        preprocessing=preprocessor_object,
+        preprocessors=built_preprocessors,
         storage=storage_object,
     )
+    mc.validate(datagrabber_object)
+
     # Fit elements
     with datagrabber_object:
         if elements is not None:
-            for t_element in elements:
+            for t_element in datagrabber_object.filter(
+                elements  # type: ignore
+            ):
                 mc.fit(datagrabber_object[t_element])
         else:
             for t_element in datagrabber_object:
@@ -213,7 +222,7 @@ def queue(
     ----------
     config : dict
         The configuration to be used for queueing the job.
-    kind : {"HTCondor", "SLURM"}
+    kind : {"HTCondor", "GNUParallelLocal"}
         The kind of job queue system to use.
     jobname : str, optional
         The name of the job (default "junifer_job").
@@ -228,13 +237,20 @@ def queue(
     Raises
     ------
     ValueError
-        If the value of ``kind`` is invalid.
+        If ``kind`` is invalid or
+        if the ``jobdir`` exists and ``overwrite = False``.
 
     """
+    valid_kind = ["HTCondor", "GNUParallelLocal"]
+    if kind not in valid_kind:
+        raise_error(
+            f"Invalid value for `kind`: {kind}, "
+            f"must be one of {valid_kind}"
+        )
+
     # Create a folder within the CWD to store the job files / config
-    cwd = Path.cwd()
-    jobdir = cwd / "junifer_jobs" / jobname
-    logger.info(f"Creating job in {jobdir.absolute()!s}")
+    jobdir = Path.cwd() / "junifer_jobs" / jobname
+    logger.info(f"Creating job directory at {jobdir.resolve()!s}")
     if jobdir.exists():
         if not overwrite:
             raise_error(
@@ -242,15 +258,16 @@ def queue(
                 "This error is raised to prevent overwriting job files "
                 "that might be scheduled but not yet executed. "
                 f"Either delete the directory {jobdir.absolute()!s} "
-                "or set overwrite=True."
+                "or set `overwrite=True.`"
             )
         else:
             logger.info(
-                f"Deleting existing job directory at {jobdir.absolute()!s}"
+                f"Deleting existing job directory at {jobdir.resolve()!s}"
             )
             shutil.rmtree(jobdir)
     jobdir.mkdir(exist_ok=True, parents=True)
 
+    # Load modules
     if "with" in config:
         to_load = config["with"]
         # If there is a list of files to load, copy and remove the path
@@ -260,15 +277,16 @@ def queue(
             to_load = [to_load]
         for item in to_load:
             if item.endswith(".py"):
-                logger.debug(f"Copying {item} to jobdir ({jobdir.absolute()})")
-                shutil.copy(item, jobdir)
+                logger.debug(f"Copying {item} to ({jobdir.resolve()!s})")
+                shutil.copy(src=item, dst=jobdir)
                 fixed_load.append(Path(item).name)
             else:
                 fixed_load.append(item)
         config["with"] = fixed_load
 
+    # Save YAML
     yaml_config = jobdir / "config.yaml"
-    logger.info(f"Writing YAML config to {yaml_config.absolute()!s}")
+    logger.info(f"Writing YAML config to {yaml_config.resolve()!s}")
     yaml.dump(config, stream=yaml_config)
 
     # Get list of elements
@@ -281,415 +299,65 @@ def queue(
             datagrabber = _get_datagrabber(config["datagrabber"])
             with datagrabber as dg:
                 elements = dg.get_elements()
-
-    # TODO: Fix typing of elements
+    # Listify elements
     if not isinstance(elements, list):
-        elements = [elements]  # type: ignore
-
-    typing.cast(List[Union[str, Tuple]], elements)
+        elements: List[Union[str, Tuple]] = [elements]
 
+    # Check job queueing system
+    adapter = None
     if kind == "HTCondor":
-        _queue_condor(
-            jobname=jobname,
-            jobdir=jobdir,
-            yaml_config=yaml_config,
-            elements=elements,  # type: ignore
-            config=config,
-            **kwargs,
+        adapter = HTCondorAdapter(
+            job_name=jobname,
+            job_dir=jobdir,
+            yaml_config_path=yaml_config,
+            elements=elements,
+            **kwargs,  # type: ignore
         )
-    elif kind == "SLURM":
-        _queue_slurm(
-            jobname=jobname,
-            jobdir=jobdir,
-            yaml_config=yaml_config,
-            elements=elements,  # type: ignore
-            config=config,
-            **kwargs,
+    elif kind == "GNUParallelLocal":
+        adapter = GnuParallelLocalAdapter(
+            job_name=jobname,
+            job_dir=jobdir,
+            yaml_config_path=yaml_config,
+            elements=elements,
+            **kwargs,  # type: ignore
         )
-    else:
-        raise_error(f"Unknown queue kind: {kind}")
 
+    adapter.prepare()  # type: ignore
     logger.info("Queue done")
 
 
-def _queue_condor(
-    jobname: str,
-    jobdir: Path,
-    yaml_config: Path,
-    elements: List[Union[str, Tuple]],
-    config: Dict,
-    env: Optional[Dict[str, str]] = None,
-    mem: str = "8G",
-    cpus: int = 1,
-    disk: str = "1G",
-    extra_preamble: str = "",
-    pre_run: Optional[str] = None,
-    verbose: str = "info",
-    collect: str = "yes",
-    submit: bool = False,
-) -> None:
-    """Submit job to HTCondor.
+def reset(config: Dict) -> None:
+    """Reset the storage and jobs directory.
 
     Parameters
     ----------
-    jobname : str
-        The name of the job.
-    jobdir : pathlib.Path
-        The path to the job directory.
-    yaml_config : pathlib.Path
-        The path to the YAML config file.
-    elements : list of str or tuple
-        Element(s) to process. Will be used to index the DataGrabber.
     config : dict
-        The configuration to be used for queueing the job.
-    env : dict, optional
-        The environment variables passed as dictionary (default None).
-    mem : str, optional
-        The size of memory (RAM) to use (default "8G").
-    cpus : int, optional
-        The number of CPU cores to use (default 1).
-    disk : str, optional
-        The size of disk (HDD or SSD) to use (default "1G").
-    extra_preamble : str, optional
-        Extra commands to pass to HTCondor (default "").
-    pre_run : str, optional
-        Extra bash commands to source before the job (default None).
-    verbose : str, optional
-        The level of verbosity (default "info").
-    collect : str, optional
-        Whether to submit "collect" task for junifer (default "yes").
-        Valid options are:
-
-            * "yes": Submit "collect" task and run even if some of the jobs
-                fail.
-            * "on_success_only": Submit "collect" task and run only if all jobs
-                succeed.
-            * "no": Do not submit "collect" task.
-
-    submit : bool, optional
-        Whether to submit the jobs. In any case, .dag files will be created
-        for submission (default False).
-
-    Raises
-    ------
-    ValueError
-        If the value of `env` is invalid.
+        The configuration to be used for resetting.
 
     """
-    logger.debug("Creating HTCondor job")
-    run_junifer_args = (
-        f"run {yaml_config.absolute()!s} "
-        f"--verbose {verbose} --element $(element)"
-    )
-    collect_junifer_args = (
-        f"collect {yaml_config.absolute()!s} --verbose {verbose} "
-    )
-
-    if not isinstance(collect, str):
-        raise_error("collect must be a string")
-
-    collect = collect.lower()
-    if collect not in ["yes", "no", "on_success_only"]:
-        raise_error(f"Invalid value for collect: {collect}")
-
-    # Set up the env_name, executable and arguments according to the
-    # environment type
-    if env is None:
-        env = {"kind": "local"}
-    if env["kind"] == "conda":
-        env_name = env["name"]
-        executable = "run_conda.sh"
-        arguments = f"{env_name} junifer"
-        exec_path = jobdir / executable
-        logger.info(f"Copying {executable} to {exec_path.absolute()!s}")
-        shutil.copy(Path(__file__).parent / "res" / executable, exec_path)
-        make_executable(exec_path)
-    elif env["kind"] == "venv":
-        env_name = env["name"]
-        executable = "run_venv.sh"
-        arguments = f"{env_name} junifer"
-        # TODO: Copy run_venv.sh to jobdir
-    elif env["kind"] == "local":
-        executable = "junifer"
-        arguments = ""
-    else:
-        raise ValueError(f'Unknown env kind: {env["kind"]}')
-
-    logger.info("Writing pre_run.sh to jobdir")
-    pre_run_fname = jobdir / "pre_run.sh"
-    with open(pre_run_fname, "w") as f:
-        f.write("#!/bin/bash\n\n")
-        f.write("# Force datalad to run in non-interactive mode\n")
-        f.write("DATALAD_UI_INTERACTIVE=false\n\n")
-        if pre_run is not None:
-            f.write(pre_run)
-    make_executable(pre_run_fname)
-
-    # Create log directory
-    log_dir = jobdir / "logs"
-    log_dir.mkdir(exist_ok=True, parents=True)
-
-    # Add preamble data
-    run_preamble = f"""
-        # The environment
-        universe = vanilla
-        getenv = True
-
-        # Resources
-        request_cpus = {cpus}
-        request_memory = {mem}
-        request_disk = {disk}
-
-        # Executable
-        initial_dir = {jobdir.absolute()!s}
-        executable = $(initial_dir)/{executable}
-        transfer_executable = False
-
-        arguments = {arguments} {run_junifer_args}
-
-        {extra_preamble}
-
-        # Logs
-        log = {log_dir.absolute()!s}/junifer_run_$(log_element).log
-        output = {log_dir.absolute()!s}/junifer_run_$(log_element).out
-        error = {log_dir.absolute()!s}/junifer_run_$(log_element).err
-        """
-
-    submit_run_fname = jobdir / f"run_{jobname}.submit"
-    submit_collect_fname = jobdir / f"collect_{jobname}.submit"
-    dag_fname = jobdir / f"{jobname}.dag"
-
-    # Write to run submit files
-    with open(submit_run_fname, "w") as submit_file:
-        submit_file.write(textwrap.dedent(run_preamble))
-        submit_file.write("queue\n")
-
-    collect_preamble = f"""
-        # The environment
-        universe = vanilla
-        getenv = True
-
-        # Resources
-        request_cpus = {cpus}
-        request_memory = {mem}
-        request_disk = {disk}
-
-        # Executable
-        initial_dir = {jobdir.absolute()!s}
-        executable = $(initial_dir)/{executable}
-        transfer_executable = False
-
-        arguments = {arguments} {collect_junifer_args}
-
-        {extra_preamble}
-
-        # Logs
-        log = {log_dir.absolute()!s}/junifer_collect.log
-        output = {log_dir.absolute()!s}/junifer_collect.out
-        error = {log_dir.absolute()!s}/junifer_collect.err
-        """
-
-    # Now create the collect submit file
-    with open(submit_collect_fname, "w") as submit_file:
-        submit_file.write(textwrap.dedent(collect_preamble))
-        submit_file.write("queue\n")
-
-    with open(dag_fname, "w") as dag_file:
-        # Get all subject and session names from file list
-        for i_job, t_elem in enumerate(elements):
-            str_elem = (
-                ",".join(t_elem) if isinstance(t_elem, tuple) else t_elem
-            )
-            log_elem = (
-                "_".join(t_elem) if isinstance(t_elem, tuple) else t_elem
-            )
-            dag_file.write(f"JOB run{i_job} {submit_run_fname}\n")
-            dag_file.write(
-                f'VARS run{i_job} element="{str_elem}" '
-                f'log_element="{log_elem}"\n\n'
-            )
-        if collect == "yes":
-            dag_file.write(f"FINAL collect {submit_collect_fname}\n")
-            collect_pre_fname = jobdir / "collect_pre.sh"
-            dag_file.write(
-                f"SCRIPT PRE collect {collect_pre_fname.as_posix()} "
-                "$DAG_STATUS\n"
-            )
-            with open(collect_pre_fname, "w") as pre_file:
-                pre_file.write("#!/bin/bash\n\n")
-                pre_file.write('if [ "${1}" == "4" ]; then\n')
-                pre_file.write("    exit 1\n")
-                pre_file.write("fi\n")
-
-            make_executable(collect_pre_fname)
-        elif collect == "on_success_only":
-            dag_file.write(f"JOB collect {submit_collect_fname}\n")
-            dag_file.write("PARENT ")
-            for i_job, _ in enumerate(elements):
-                dag_file.write(f"run{i_job} ")
-            dag_file.write("CHILD collect\n\n")
-
-    # Submit job(s)
-    if submit is True:
-        logger.info("Submitting HTCondor job")
-        subprocess.run(["condor_submit_dag", dag_fname])
-        logger.info("HTCondor job submitted")
-    else:
-        cmd = f"condor_submit_dag {dag_fname.absolute()!s}"
-        logger.info(
-            f"HTCondor job files created, to submit the job, run `{cmd}`"
+    # Fetch storage
+    storage = config["storage"]
+    storage_uri = Path(storage["uri"])
+    logger.info(f"Deleting {storage_uri.resolve()!s}")
+    # Delete storage; will be str
+    if storage_uri.exists():
+        # Delete files in the directory
+        for file in storage_uri.iterdir():
+            file.unlink(missing_ok=True)
+        # Remove directory
+        storage_uri.parent.rmdir()
+
+    # Fetch job name (if present)
+    if config.get("queue") is not None:
+        queue = config["queue"]
+        job_dir = (
+            Path.cwd()
+            / "junifer_jobs"
+            / (queue.get("jobname") or "junifer_job")
         )
-
-
-def _queue_slurm(
-    jobname: str,
-    jobdir: Path,
-    yaml_config: Path,
-    elements: List[Union[str, Tuple]],
-    config: Dict,
-) -> None:
-    """Submit job to SLURM.
-
-    Parameters
-    ----------
-    jobname : str
-        The name of the job.
-    jobdir : pathlib.Path
-        The path to the job directory.
-    yaml_config : pathlib.Path
-        The path to the YAML config file.
-    elements : str or tuple or list[str or tuple], optional
-        Element(s) to process. Will be used to index the DataGrabber
-        (default None).
-    config : dict
-        The configuration to be used for queueing the job.
-    """
-    pass
-    # logger.debug("Creating SLURM job")
-    # run_junifer_args = (
-    #     f"run {str(yaml_config.absolute())} "
-    #     f"--verbose {verbose} --element $(element)"
-    # )
-    # collect_junifer_args = \
-    #     f"collect {str(yaml_config.absolute())} --verbose {verbose} "
-
-    # # Set up the env_name, executable and arguments according to the
-    # # environment type
-    # if env is None:
-    #     env = {
-    #         "kind": "local",
-    #     }
-    # if env["kind"] == "conda":
-    #     env_name = env["name"]
-    #     executable = "run_conda.sh"
-    #     arguments = f"{env_name} junifer"
-    #     # TODO: Copy run_conda.sh to jobdir
-    #     exec_path = jobdir / executable
-    #     shutil.copy(Path(__file__).parent / "res" / executable, exec_path)
-    #     make_executable(exec_path)
-    # elif env["kind"] == "venv":
-    #     env_name = env["name"]
-    #     executable = "run_venv.sh"
-    #     arguments = f"{env_name} junifer"
-    #     # TODO: Copy run_venv.sh to jobdir
-    # elif env["kind"] == "local":
-    #     executable = "junifer"
-    #     arguments = ""
-    # else:
-    #     raise ValueError(f"Unknown env kind: {env['kind']}")
-
-    # # Create log directory
-    # log_dir = jobdir / 'logs'
-    # log_dir.mkdir(exist_ok=True, parents=True)
-
-    # # Add preamble data
-    # run_preamble = f"""
-    #     #!/bin/bash
-
-    #     #SBATCH --job-name={}
-    #     #SBATCH --account={}
-    #     #SBATCH --partition={}
-    #     #SBATCH --time={}
-    #     #SBATCH --ntasks={}
-    #     #SBATCH --cpus-per-task={cpus}
-    #     #SBATCH --mem-per-cpu={mem}
-    #     #SBATCH --mail-type={}
-    #     #SBATCH --mail-user={}
-    #     #SBATCH --output={}
-    #     #SBATCH --error={}
-
-    #     # Executable
-    #     initial_dir = {str(jobdir.absolute())}
-    #     executable = $(initial_dir)/{executable}
-    #     transfer_executable = False
-
-    #     arguments = {arguments} {run_junifer_args}
-
-    #     {extra_preamble}
-
-    #     # Logs
-    #     log = {str(log_dir.absolute())}/junifer_run_$(element).log
-    #     output = {str(log_dir.absolute())}/junifer_run_$(element).out
-    #     error = {str(log_dir.absolute())}/junifer_run_$(element).err
-    #     """
-
-    # submit_run_fname = jobdir / f'run_{jobname}.sh'
-    # submit_collect_fname = jobdir / f'collect_{jobname}.sh'
-
-    # # Write to run submit files
-    # with open(submit_run_fname, 'w') as submit_file:
-    #     submit_file.write(run_preamble)
-    #     submit_file.write('queue\n')
-
-    # collect_preamble = f"""
-    #     # The environment
-    #     universe = vanilla
-    #     getenv = True
-
-    #     # Resources
-    #     request_cpus = {cpus}
-    #     request_memory = {mem}
-    #     request_disk = {disk}
-
-    #     # Executable
-    #     initial_dir = {str(jobdir.absolute())}
-    #     executable = $(initial_dir)/{executable}
-    #     transfer_executable = False
-
-    #     arguments = {arguments} {collect_junifer_args}
-
-    #     {extra_preamble}
-
-    #     # Logs
-    #     log = {str(log_dir.absolute())}/junifer_collect.log
-    #     output = {str(log_dir.absolute())}/junifer_collect.out
-    #     error = {str(log_dir.absolute())}/junifer_collect.err
-    #     """
-
-    # # Now create the collect submit file
-    # with open(submit_collect_fname, 'w') as submit_file:
-    #     submit_file.write(collect_preamble)  # Eval preamble here
-    #     submit_file.write('queue\n')
-
-    # with open(dag_fname, 'w') as dag_file:
-    #     # Get all subject and session names from file list
-    #     for i_job, t_elem in enumerate(elements):
-    #         dag_file.write(f'JOB run{i_job} {submit_run_fname}\n')
-    #         dag_file.write(f'VARS run{i_job} element="{t_elem}"\n\n')
-    #     if collect is True:
-    #         dag_file.write(f'JOB collect {submit_collect_fname}\n')
-    #         dag_file.write('PARENT ')
-    #         for i_job, _t_elem in enumerate(elements):
-    #             dag_file.write(f'run{i_job} ')
-    #         dag_file.write('CHILD collect\n\n')
-
-    # # Submit job(s)
-    # if submit is True:
-    #     logger.info('Submitting SLURM job')
-    #     subprocess.run(['condor_submit_dag', dag_fname])
-    #     logger.info('HTCondor SLURM submitted')
-    # else:
-    #     cmd = f"condor_submit_dag {str(dag_fname.absolute())}"
-    #     logger.info(
-    #         f"SLURM job files created, to submit the job, run `{cmd}`"
-    #     )
+        logger.info(f"Deleting job directory at {job_dir.resolve()!s}")
+        if job_dir.exists():
+            # Remove files and directories
+            shutil.rmtree(job_dir)
+            # Remove directory
+            job_dir.parent.rmdir()

junifer/api/parser.py

@@ -50,6 +50,8 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
         # Convert load modules to list
         if not isinstance(to_load, list):
             to_load = [to_load]
+        # Initialize list to have absolute paths for custom modules
+        final_to_load = []
         for t_module in to_load:
             if t_module.endswith(".py"):
                 logger.debug(f"Importing file: {t_module}")
@@ -65,9 +67,18 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
                 module = importlib.util.module_from_spec(spec)  # type: ignore
                 sys.modules[t_module] = module
                 spec.loader.exec_module(module)  # type: ignore
+                # Add absolute path to final list
+                final_to_load.append(str(file_path.resolve()))
             else:
                 logger.info(f"Importing module: {t_module}")
                 importlib.import_module(t_module)
+                # Add module to final list
+                final_to_load.append(t_module)
+
+        # Replace modules to be loaded so that custom modules will take the
+        # absolute path. This was not the case as found in #224. Similar thing
+        # is done with the storage URI below.
+        contents["with"] = final_to_load
 
     # Compute path for the URI parameter in storage files that are relative
     # This is a tricky thing that appeared in #127. The problem is that
@@ -88,4 +99,17 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
                 contents["storage"]["uri"] = str(
                     (filepath.parent / uri_path).resolve()
                 )
+
+    # Allow relative path if queue env kind is venv; same motivation as above
+    if "queue" in contents:
+        if "env" in contents["queue"]:
+            if "venv" == contents["queue"]["env"]["kind"]:
+                # Check if the env name is relative
+                venv_path = Path(contents["queue"]["env"]["name"])
+                if not venv_path.is_absolute():
+                    # Compute the absolute path
+                    contents["queue"]["env"]["name"] = str(
+                        (filepath.parent / venv_path).resolve()
+                    )
+
     return contents

junifer/api/queue_context/__init__.py

@@ -0,0 +1,8 @@
+"""Provide imports for queue context sub-package."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from .queue_context_adapter import QueueContextAdapter
+from .htcondor_adapter import HTCondorAdapter
+from .gnu_parallel_local_adapter import GnuParallelLocalAdapter