junifer 0.0.5.dev240__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/__init__.py

@@ -4,36 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from . import (
-    api,
-    configs,
-    data,
-    datagrabber,
-    datareader,
-    markers,
-    pipeline,
-    preprocess,
-    stats,
-    storage,
-    utils,
-    external,
-    onthefly,
-)
-from ._version import __version__
+import lazy_loader as lazy
 
 
-__all__ = [
-    "api",
-    "configs",
-    "data",
-    "datagrabber",
-    "datareader",
-    "markers",
-    "pipeline",
-    "preprocess",
-    "stats",
-    "storage",
-    "utils",
-    "external",
-    "onthefly",
-]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/__init__.pyi

@@ -0,0 +1,37 @@
+__all__ = [
+    "api",
+    "configs",
+    "data",
+    "datagrabber",
+    "datareader",
+    "markers",
+    "pipeline",
+    "preprocess",
+    "stats",
+    "storage",
+    "utils",
+    "external",
+    "onthefly",
+    "testing",
+    "typing",
+    "__version__",
+]
+
+from . import (
+    api,
+    configs,
+    data,
+    datagrabber,
+    datareader,
+    markers,
+    pipeline,
+    preprocess,
+    stats,
+    storage,
+    utils,
+    external,
+    testing,
+    typing,
+    onthefly,
+)
+from ._version import __version__

junifer/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.5.dev240'
-__version_tuple__ = version_tuple = (0, 0, 5, 'dev240')
+__version__ = version = '0.0.6'
+__version_tuple__ = version_tuple = (0, 0, 6)

junifer/api/__init__.py

@@ -1,12 +1,10 @@
-"""Public API and CLI components."""
+"""Public API components."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from . import decorators
-from .cli import cli
-from .functions import collect, queue, run
+import lazy_loader as lazy
 
 
-__all__ = ["decorators", "cli", "collect", "queue", "run"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/api/__init__.pyi

@@ -0,0 +1,4 @@
+__all__ = ["decorators", "collect", "queue", "run", "reset", "list_elements"]
+
+from . import decorators
+from .functions import collect, list_elements, reset, run, queue

junifer/api/decorators.py

@@ -1,25 +1,25 @@
-"""Provide decorators for api."""
+"""Provide API decorators."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Type
 
-from ..pipeline.registry import register
+from ..pipeline import PipelineComponentRegistry
+from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
 
 
 __all__ = [
     "register_datagrabber",
     "register_datareader",
-    "register_preprocessor",
     "register_marker",
+    "register_preprocessor",
     "register_storage",
 ]
 
 
-def register_datagrabber(klass: Type) -> Type:
+def register_datagrabber(klass: DataGrabberLike) -> DataGrabberLike:
     """Register DataGrabber.
 
     Registers the DataGrabber so it can be used by name.
@@ -39,15 +39,14 @@ def register_datagrabber(klass: Type) -> Type:
     It should only be used as a decorator.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="datagrabber",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
 
 
-def register_datareader(klass: Type) -> Type:
+def register_datareader(klass: type) -> type:
     """Register DataReader.
 
     Registers the DataReader so it can be used by name.
@@ -67,15 +66,14 @@ def register_datareader(klass: Type) -> Type:
     It should only be used as a decorator.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="datareader",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
 
 
-def register_preprocessor(klass: Type) -> Type:
+def register_preprocessor(klass: PreprocessorLike) -> PreprocessorLike:
     """Preprocessor registration decorator.
 
     Registers the preprocessor so it can be used by name.
@@ -91,15 +89,14 @@ def register_preprocessor(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="preprocessing",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
 
 
-def register_marker(klass: Type) -> Type:
+def register_marker(klass: MarkerLike) -> MarkerLike:
     """Marker registration decorator.
 
     Registers the marker so it can be used by name.
@@ -115,15 +112,14 @@ def register_marker(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="marker",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
 
 
-def register_storage(klass: Type) -> Type:
+def register_storage(klass: StorageLike) -> StorageLike:
     """Storage registration decorator.
 
     Registers the storage so it can be used by name.
@@ -139,9 +135,8 @@ def register_storage(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="storage",
-        name=klass.__name__,
         klass=klass,
     )
     return klass

junifer/api/functions.py

@@ -1,4 +1,4 @@
-"""Provide functions for cli."""
+"""Provide API functions."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>
@@ -7,26 +7,33 @@
 
 import os
 import shutil
-import typing
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
-
-from ..datagrabber.base import BaseDataGrabber
-from ..markers.base import BaseMarker
-from ..markers.collection import MarkerCollection
-from ..pipeline import WorkDirManager
-from ..pipeline.registry import build
-from ..preprocess.base import BasePreprocessor
-from ..storage.base import BaseFeatureStorage
-from ..utils import logger, raise_error
-from .queue_context import GnuParallelLocalAdapter, HTCondorAdapter
-from .utils import yaml
-
-
-__all__ = ["run", "collect", "queue", "reset", "list_elements"]
-
-
-def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber:
+from typing import Optional, Union
+
+from ..api.queue_context import GnuParallelLocalAdapter, HTCondorAdapter
+from ..datagrabber import BaseDataGrabber
+from ..markers import BaseMarker
+from ..pipeline import (
+    MarkerCollection,
+    PipelineComponentRegistry,
+    WorkDirManager,
+)
+from ..preprocess import BasePreprocessor
+from ..storage import BaseFeatureStorage
+from ..typing import (
+    DataGrabberLike,
+    Elements,
+    MarkerLike,
+    PreprocessorLike,
+    StorageLike,
+)
+from ..utils import logger, raise_error, warn_with_log, yaml
+
+
+__all__ = ["collect", "list_elements", "queue", "reset", "run"]
+
+
+def _get_datagrabber(datagrabber_config: dict) -> DataGrabberLike:
     """Get DataGrabber.
 
     Parameters
@@ -40,57 +47,93 @@ def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber:
         The DataGrabber.
 
     """
-    datagrabber_params = datagrabber_config.copy()
-    datagrabber_kind = datagrabber_params.pop("kind")
-    datagrabber = build(
+    return PipelineComponentRegistry().build_component_instance(
         step="datagrabber",
-        name=datagrabber_kind,
+        name=datagrabber_config.pop("kind"),
         baseclass=BaseDataGrabber,
-        init_params=datagrabber_params,
+        init_params=datagrabber_config,
     )
-    datagrabber = typing.cast(BaseDataGrabber, datagrabber)
-    return datagrabber
 
 
-def _get_preprocessor(preprocessing_config: Dict) -> BasePreprocessor:
-    """Get preprocessor.
+def _get_preprocessor(preprocessing_config: dict) -> PreprocessorLike:
+    """Get Preprocessor.
 
     Parameters
     ----------
     preprocessing_config : dict
-        The config to get the preprocessor using.
+        The config to get the Preprocessor using.
 
     Returns
     -------
     dict
-        The preprocessor.
+        The Preprocessor.
 
     """
-    preprocessor_params = preprocessing_config.copy()
-    preprocessor_kind = preprocessor_params.pop("kind")
-    preprocessor = build(
+    return PipelineComponentRegistry().build_component_instance(
         step="preprocessing",
-        name=preprocessor_kind,
+        name=preprocessing_config.pop("kind"),
         baseclass=BasePreprocessor,
-        init_params=preprocessor_params,
+        init_params=preprocessing_config,
+    )
+
+
+def _get_marker(marker_config: dict) -> MarkerLike:
+    """Get Marker.
+
+    Parameters
+    ----------
+    marker_config : dict
+        The config to get the Marker using.
+
+    Returns
+    -------
+    object
+        The Marker.
+
+    """
+    return PipelineComponentRegistry().build_component_instance(
+        step="marker",
+        name=marker_config.pop("kind"),
+        baseclass=BaseMarker,
+        init_params=marker_config,
+    )
+
+
+def _get_storage(storage_config: dict) -> StorageLike:
+    """Get Storage.
+
+    Parameters
+    ----------
+    storage_config : dict
+        The config to get the Storage using.
+
+    Returns
+    -------
+    dict
+        The Storage.
+
+    """
+    return PipelineComponentRegistry().build_component_instance(
+        step="storage",
+        name=storage_config.pop("kind"),
+        baseclass=BaseFeatureStorage,
+        init_params=storage_config,
     )
-    preprocessor = typing.cast(BasePreprocessor, preprocessor)
-    return preprocessor
 
 
 def run(
-    workdir: Union[str, Path],
-    datagrabber: Dict,
-    markers: List[Dict],
-    storage: Dict,
-    preprocessors: Optional[List[Dict]] = None,
-    elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None,
+    workdir: Union[str, Path, dict],
+    datagrabber: dict,
+    markers: list[dict],
+    storage: dict,
+    preprocessors: Optional[list[dict]] = None,
+    elements: Optional[Elements] = None,
 ) -> None:
     """Run the pipeline on the selected element.
 
     Parameters
     ----------
-    workdir : str or pathlib.Path
+    workdir : str or pathlib.Path or dict
         Directory where the pipeline will be executed.
     datagrabber : dict
         DataGrabber to use. Must have a key ``kind`` with the kind of
@@ -106,60 +149,58 @@ def run(
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
         constructor.
-    preprocessors : list of dict, optional
+    preprocessors : list of dict or None, optional
         List of preprocessors to use. Each preprocessor is a dict with at
         least a key ``kind`` specifying the preprocessor to use. All other keys
         are passed to the preprocessor constructor (default None).
-    elements : str or tuple or list of str or tuple, optional
+    elements : list or None, optional
         Element(s) to process. Will be used to index the DataGrabber
         (default None).
 
+    Raises
+    ------
+    ValueError
+        If ``workdir.cleanup=False`` when ``len(elements) > 1``.
+    RuntimeError
+        If invalid element selectors are found.
+
     """
-    # Convert str to Path
-    if isinstance(workdir, str):
-        workdir = Path(workdir)
-    # Initiate working directory manager
-    WorkDirManager(workdir)
+    # Conditional to handle workdir config
+    if isinstance(workdir, (str, Path)):
+        if isinstance(workdir, str):
+            workdir = {"workdir": Path(workdir), "cleanup": True}
+        else:
+            workdir = {"workdir": workdir, "cleanup": True}
+    elif isinstance(workdir, dict):
+        workdir["workdir"] = workdir.pop("path")
 
-    if not isinstance(elements, list) and elements is not None:
-        elements = [elements]
+    # Initiate working directory manager with correct variation
+    if not workdir["cleanup"]:
+        if elements is None or len(elements) > 1:
+            raise_error(
+                "Cannot disable `workdir.cleanup` as "
+                f"{len(elements) if elements is not None else 'all'} "
+                "elements will be processed"
+            )
+    WorkDirManager(**workdir)
 
     # Get datagrabber to use
-    datagrabber_object = _get_datagrabber(datagrabber)
+    datagrabber_object = _get_datagrabber(datagrabber.copy())
 
-    # Copy to avoid changing the original dict
-    _markers = [x.copy() for x in markers]
-    built_markers = []
-    for t_marker in _markers:
-        kind = t_marker.pop("kind")
-        t_m = build(
-            step="marker",
-            name=kind,
-            baseclass=BaseMarker,
-            init_params=t_marker,
-        )
-        built_markers.append(t_m)
+    # Get markers to use
+    built_markers = [_get_marker(marker) for marker in markers.copy()]
 
     # Get storage engine to use
-    storage_params = storage.copy()
-    storage_kind = storage_params.pop("kind")
-    if "single_output" not in storage_params:
-        storage_params["single_output"] = False
-    storage_object = build(
-        step="storage",
-        name=storage_kind,
-        baseclass=BaseFeatureStorage,
-        init_params=storage_params,
-    )
-    storage_object = typing.cast(BaseFeatureStorage, storage_object)
+    if "single_output" not in storage:
+        storage["single_output"] = False
+    storage_object = _get_storage(storage.copy())
 
     # Get preprocessor to use (if provided)
     if preprocessors is not None:
-        _preprocessors = [x.copy() for x in preprocessors]
-        built_preprocessors = []
-        for preprocessor in _preprocessors:
-            preprocessor_object = _get_preprocessor(preprocessor)
-            built_preprocessors.append(preprocessor_object)
+        built_preprocessors = [
+            _get_preprocessor(preprocessor)
+            for preprocessor in preprocessors.copy()
+        ]
     else:
         built_preprocessors = None
 
@@ -169,21 +210,34 @@ def run(
         preprocessors=built_preprocessors,
         storage=storage_object,
     )
+    # Validate the marker collection for the datagrabber
     mc.validate(datagrabber_object)
 
     # Fit elements
     with datagrabber_object:
         if elements is not None:
-            for t_element in datagrabber_object.filter(
-                elements  # type: ignore
-            ):
+            # Keep track of valid selectors
+            valid_elements = []
+            for t_element in datagrabber_object.filter(elements):
+                valid_elements.append(t_element)
                 mc.fit(datagrabber_object[t_element])
+            # Compute invalid selectors
+            invalid_elements = set(elements) - set(valid_elements)
+            # Report if invalid selectors are found
+            if invalid_elements:
+                raise_error(
+                    msg=(
+                        "The following element selectors are invalid:\n"
+                        f"{invalid_elements}"
+                    ),
+                    klass=RuntimeError,
+                )
         else:
             for t_element in datagrabber_object:
                 mc.fit(datagrabber_object[t_element])
 
 
-def collect(storage: Dict) -> None:
+def collect(storage: dict) -> None:
     """Collect and store data.
 
     Parameters
@@ -194,31 +248,23 @@ def collect(storage: Dict) -> None:
         constructor.
 
     """
-    storage_params = storage.copy()
-    storage_kind = storage_params.pop("kind")
-    logger.info(f"Collecting data using {storage_kind}")
-    logger.debug(f"\tStorage params: {storage_params}")
-    if "single_output" not in storage_params:
-        storage_params["single_output"] = False
-    storage_object = build(
-        step="storage",
-        name=storage_kind,
-        baseclass=BaseFeatureStorage,
-        init_params=storage_params,
-    )
-    storage_object = typing.cast(BaseFeatureStorage, storage_object)
+    logger.info(f"Collecting data using {storage['kind']}")
+    logger.debug(f"\tStorage params: {storage}")
+    if "single_output" not in storage:
+        storage["single_output"] = False
+    storage_object = _get_storage(storage.copy())
     logger.debug("Running storage.collect()")
     storage_object.collect()
     logger.info("Collect done")
 
 
 def queue(
-    config: Dict,
+    config: dict,
     kind: str,
     jobname: str = "junifer_job",
     overwrite: bool = False,
-    elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None,
-    **kwargs: Union[str, int, bool, Dict, Tuple, List],
+    elements: Optional[Elements] = None,
+    **kwargs: Union[str, int, bool, dict, tuple, list],
 ) -> None:
     """Queue a job to be executed later.
 
@@ -232,7 +278,7 @@ def queue(
         The name of the job (default "junifer_job").
     overwrite : bool, optional
         Whether to overwrite if job directory already exists (default False).
-    elements : str or tuple or list of str or tuple, optional
+    elements : list or None, optional
         Element(s) to process. Will be used to index the DataGrabber
         (default None).
     **kwargs : dict
@@ -271,6 +317,16 @@ def queue(
             shutil.rmtree(jobdir)
     jobdir.mkdir(exist_ok=True, parents=True)
 
+    # Check workdir config
+    if "workdir" in config:
+        if isinstance(config["workdir"], dict):
+            if not config["workdir"]["cleanup"]:
+                warn_with_log(
+                    "`workdir.cleanup` will be set to True when queueing"
+                )
+            # Set cleanup
+            config["workdir"]["cleanup"] = True
+
     # Load modules
     if "with" in config:
         to_load = config["with"]
@@ -305,7 +361,7 @@ def queue(
                 elements = dg.get_elements()
     # Listify elements
     if not isinstance(elements, list):
-        elements: List[Union[str, Tuple]] = [elements]
+        elements: Elements = [elements]
 
     # Check job queueing system
     adapter = None
@@ -330,7 +386,7 @@ def queue(
     logger.info("Queue done")
 
 
-def reset(config: Dict) -> None:
+def reset(config: dict) -> None:
     """Reset the storage and jobs directory.
 
     Parameters
@@ -369,8 +425,8 @@ def reset(config: Dict) -> None:
 
 
 def list_elements(
-    datagrabber: Dict,
-    elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None,
+    datagrabber: dict,
+    elements: Optional[Elements] = None,
 ) -> str:
     """List elements of the datagrabber filtered using `elements`.
 
@@ -380,7 +436,7 @@ def list_elements(
         DataGrabber to index. Must have a key ``kind`` with the kind of
         DataGrabber to use. All other keys are passed to the DataGrabber
         constructor.
-    elements : str or tuple or list of str or tuple, optional
+    elements : list or None, optional
         Element(s) to filter using. Will be used to index the DataGrabber
         (default None).
 

junifer/api/queue_context/__init__.py

@@ -3,9 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .queue_context_adapter import QueueContextAdapter
-from .htcondor_adapter import HTCondorAdapter
-from .gnu_parallel_local_adapter import GnuParallelLocalAdapter
+import lazy_loader as lazy
 
 
-__all__ = ["QueueContextAdapter", "HTCondorAdapter", "GnuParallelLocalAdapter"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/api/queue_context/__init__.pyi

@@ -0,0 +1,5 @@
+__all__ = ["QueueContextAdapter", "HTCondorAdapter", "GnuParallelLocalAdapter"]
+
+from .queue_context_adapter import QueueContextAdapter
+from .htcondor_adapter import HTCondorAdapter
+from .gnu_parallel_local_adapter import GnuParallelLocalAdapter