looper 1.7.0a1__py3-none-any.whl → 2.0.0__py3-none-any.whl

looper/conductor.py

@@ -4,9 +4,12 @@ import importlib
 import logging
 import os
 import subprocess
+import signal
+import psutil
+import sys
 import time
 import yaml
-from copy import copy, deepcopy
+from math import ceil
 from json import loads
 from subprocess import check_output
 from typing import *
@@ -18,14 +21,19 @@ from jinja2.exceptions import UndefinedError
 from peppy.const import CONFIG_KEY, SAMPLE_NAME_ATTR, SAMPLE_YAML_EXT
 from peppy.exceptions import RemoteYAMLError
 from pipestat import PipestatError
-from ubiquerg import expandpath, is_command_callable
+from ubiquerg import expandpath
 from yaml import dump
-from yacman import YAMLConfigManager
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
 
 from .const import *
-from .exceptions import JobSubmissionException, SampleFailedException
+from .exceptions import JobSubmissionException
 from .processed_project import populate_sample_paths
-from .utils import fetch_sample_flags, jinja_render_template_strictly
+from .utils import (
+    fetch_sample_flags,
+    jinja_render_template_strictly,
+    expand_nested_var_templates,
+)
+from .const import PipelineLevel
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -84,11 +92,23 @@ def _get_yaml_path(namespaces, template_key, default_name_appendix="", filename=
 
 def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict):
     """
-    This is run at the project level, not at the sample level.
+    This writes a combined configuration file to be passed to a PipestatManager.
+    :param str looper_pipestat_config_path: path to the created pipestat configuration file
+    :param dict pipestat_config_dict: the dict containing key value pairs to be written to the pipestat configutation
+    return bool
     """
+
+    if not os.path.exists(os.path.dirname(looper_pipestat_config_path)):
+        try:
+            os.makedirs(os.path.dirname(looper_pipestat_config_path))
+        except FileExistsError:
+            pass
+
     with open(looper_pipestat_config_path, "w") as f:
         yaml.dump(pipestat_config_dict, f)
-    print(f"Initialized looper config file: {looper_pipestat_config_path}")
+    _LOGGER.debug(
+        msg=f"Initialized pipestat config file: {looper_pipestat_config_path}"
+    )
 
     return True
 
@@ -132,6 +152,7 @@ class SubmissionConductor(object):
         compute_variables=None,
         max_cmds=None,
         max_size=None,
+        max_jobs=None,
         automatic=True,
         collate=False,
     ):
@@ -166,14 +187,20 @@ class SubmissionConductor(object):
             include in a single job script.
         :param int | float | NoneType max_size: Upper bound on total file
             size of inputs used by the commands lumped into single job script.
+        :param int | float | NoneType max_jobs: Upper bound on total number of jobs to
+            group samples for submission.
         :param bool automatic: Whether the submission should be automatic once
             the pool reaches capacity.
         :param bool collate: Whether a collate job is to be submitted (runs on
             the project level, rather that on the sample level)
         """
         super(SubmissionConductor, self).__init__()
+
         self.collate = collate
         self.section_key = PROJECT_PL_KEY if self.collate else SAMPLE_PL_KEY
+        self.pipeline_interface_type = (
+            "project_interface" if self.collate else "sample_interface"
+        )
         self.pl_iface = pipeline_interface
         self.pl_name = self.pl_iface.pipeline_name
         self.prj = prj
@@ -193,6 +220,7 @@ class SubmissionConductor(object):
         self._curr_size = 0
         self._failed_sample_names = []
         self._curr_skip_pool = []
+        self.process_id = None  # this is used for currently submitted subprocess
 
         if self.extra_pipe_args:
             _LOGGER.debug(
@@ -200,6 +228,16 @@ class SubmissionConductor(object):
                 "{}".format(self.extra_pipe_args)
             )
 
+        if max_jobs:
+            if max_jobs == 0 or max_jobs < 0:
+                raise ValueError(
+                    "If specified, max job command count must be a positive integer, greater than zero."
+                )
+
+            num_samples = len(self.prj.samples)
+            samples_per_job = num_samples / max_jobs
+            max_cmds = ceil(samples_per_job)
+
         if not self.collate:
             self.automatic = automatic
             if max_cmds is None and max_size is None:
@@ -247,8 +285,12 @@ class SubmissionConductor(object):
 
         :param bool frorce: whether to force the project submission (ignore status/flags)
         """
+        psms = {}
         if self.prj.pipestat_configured_project:
-            psm = self.prj.get_pipestat_managers(project_level=True)[self.pl_name]
+            for piface in self.prj.project_pipeline_interfaces:
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
+                    psms[piface.psm.pipeline_name] = piface.psm
+            psm = psms[self.pl_name]
             status = psm.get_status()
             if not force and status is not None:
                 _LOGGER.info(f"> Skipping project. Determined status: {status}")
@@ -274,12 +316,11 @@ class SubmissionConductor(object):
             )
         )
         if self.prj.pipestat_configured:
-            psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
-            sample_statuses = psms[self.pl_name].get_status(
+            sample_statuses = self.pl_iface.psm.get_status(
                 record_identifier=sample.sample_name
             )
             if sample_statuses == "failed" and rerun is True:
-                psms[self.pl_name].set_status(
+                self.pl_iface.psm.set_status(
                     record_identifier=sample.sample_name, status_identifier="waiting"
                 )
                 sample_statuses = "waiting"
@@ -289,23 +330,27 @@ class SubmissionConductor(object):
 
         use_this_sample = True  # default to running this sample
         msg = None
+        if rerun and sample_statuses == []:
+            msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found."
+            use_this_sample = False
         if sample_statuses:
             status_str = ", ".join(sample_statuses)
             failed_flag = any("failed" in x for x in sample_statuses)
+            waiting_flag = any("waiting" in x for x in sample_statuses)
             if self.ignore_flags:
                 msg = f"> Found existing status: {status_str}. Ignoring."
             else:  # this pipeline already has a status
                 msg = f"> Found existing status: {status_str}. Skipping sample."
-                if failed_flag:
+                if failed_flag and not rerun:
                     msg += " Use rerun to ignore failed status."  # help guidance
                 use_this_sample = False
             if rerun:
                 # Rescue the sample if rerun requested, and failed flag is found
-                if failed_flag:
-                    msg = f"> Re-running failed sample. Status: {status_str}"
+                if failed_flag or waiting_flag:
+                    msg = f"> Re-running sample. Status: {status_str}"
                     use_this_sample = True
                 else:
-                    msg = f"> Skipping sample because rerun requested, but no failed flag found. Status: {status_str}"
+                    msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found. Status: {status_str}"
                     use_this_sample = False
         if msg:
             _LOGGER.info(msg)
@@ -358,6 +403,10 @@ class SubmissionConductor(object):
             not for dry run)
         """
         submitted = False
+
+        # Override signal handler so that Ctrl+C can be used to gracefully terminate child process
+        signal.signal(signal.SIGINT, self._signal_int_handler)
+
         if not self._pool:
             _LOGGER.debug("No submission (no pooled samples): %s", self.pl_name)
             # submitted = False
@@ -386,9 +435,10 @@ class SubmissionConductor(object):
                 submission_command = "{} {}".format(sub_cmd, script)
                 # Capture submission command return value so that we can
                 # intercept and report basic submission failures; #167
-                try:
-                    subprocess.check_call(submission_command, shell=True)
-                except subprocess.CalledProcessError:
+                process = subprocess.Popen(submission_command, shell=True)
+                self.process_id = process.pid
+                process.wait()
+                if process.returncode != 0:
                     fails = (
                         "" if self.collate else [s.sample_name for s in self._samples]
                     )
@@ -455,6 +505,87 @@ class SubmissionConductor(object):
             # name concordant with 1-based, not 0-based indexing.
             return "lump{}".format(self._num_total_job_submissions + 1)
 
+    def _signal_int_handler(self, signal, frame):
+        """
+        For catching interrupt (Ctrl +C) signals. Fails gracefully.
+        """
+        signal_type = "SIGINT"
+        self._generic_signal_handler(signal_type)
+
+    def _generic_signal_handler(self, signal_type):
+        """
+        Function for handling both SIGTERM and SIGINT
+        """
+        message = "Received " + signal_type + ". Failing gracefully..."
+        _LOGGER.warning(msg=message)
+
+        self._terminate_current_subprocess()
+
+        sys.exit(1)
+
+    def _terminate_current_subprocess(self):
+        """This terminates the current sub process associated with self.process_id"""
+
+        def pskill(proc_pid, sig=signal.SIGINT):
+            parent_process = psutil.Process(proc_pid)
+            for child_proc in parent_process.children(recursive=True):
+                child_proc.send_signal(sig)
+            parent_process.send_signal(sig)
+
+        if self.process_id is None:
+            return
+
+        # Gently wait for the subprocess before attempting to kill it
+        sys.stdout.flush()
+        still_running = self._attend_process(psutil.Process(self.process_id), 0)
+        sleeptime = 0.25
+        time_waiting = 0
+
+        while still_running and time_waiting < 3:
+            try:
+                if time_waiting > 2:
+                    pskill(self.process_id, signal.SIGKILL)
+                elif time_waiting > 1:
+                    pskill(self.process_id, signal.SIGTERM)
+                else:
+                    pskill(self.process_id, signal.SIGINT)
+
+            except OSError:
+                # This would happen if the child process ended between the check
+                # and the next kill step
+                still_running = False
+                time_waiting = time_waiting + sleeptime
+
+            # Now see if it's still running
+            time_waiting = time_waiting + sleeptime
+            if not self._attend_process(psutil.Process(self.process_id), sleeptime):
+                still_running = False
+
+        if still_running:
+            _LOGGER.warning(f"Unable to halt child process: {self.process_id}")
+        else:
+            if time_waiting > 0:
+                note = f"terminated after {time_waiting} sec"
+            else:
+                note = "was already terminated"
+            _LOGGER.warning(msg=f"Child process {self.process_id} {note}.")
+
+    def _attend_process(self, proc, sleeptime):
+        """
+        Waits on a process for a given time to see if it finishes, returns True
+        if it's still running after the given time or False as soon as it
+        returns.
+
+        :param psutil.Process proc: Process object opened by psutil.Popen()
+        :param float sleeptime: Time to wait
+        :return bool: True if process is still running; otherwise false
+        """
+        try:
+            proc.wait(timeout=int(sleeptime))
+        except psutil.TimeoutExpired:
+            return True
+        return False
+
     def _jobname(self, pool):
         """Create the name for a job submission."""
         return "{}_{}".format(self.pl_iface.pipeline_name, self._sample_lump_name(pool))
@@ -514,12 +645,7 @@ class SubmissionConductor(object):
         :return yacman.YAMLConfigManager: pipestat namespace
         """
         try:
-            psms = (
-                self.prj.get_pipestat_managers(sample_name)
-                if sample_name
-                else self.prj.get_pipestat_managers(project_level=True)
-            )
-            psm = psms[self.pl_iface.pipeline_name]
+            psm = self.pl_iface.psm
         except (PipestatError, AttributeError) as e:
             # pipestat section faulty or not found in project.looper or sample
             # or project is missing required pipestat attributes
@@ -534,6 +660,8 @@ class SubmissionConductor(object):
                 "results_file": psm.file,
                 "record_identifier": psm.record_identifier,
                 "config_file": psm.config_path,
+                "output_schema": psm.cfg["_schema_path"],
+                "pephub_path": psm.cfg["pephub_path"],
             }
             filtered_namespace = {k: v for k, v in full_namespace.items() if v}
             return YAMLConfigManager(filtered_namespace)
@@ -557,7 +685,11 @@ class SubmissionConductor(object):
             pipeline=self.pl_iface,
             compute=self.prj.dcc.compute,
         )
-        templ = self.pl_iface["command_template"]
+
+        if self.pipeline_interface_type is None:
+            templ = self.pl_iface["command_template"]
+        else:
+            templ = self.pl_iface[self.pipeline_interface_type]["command_template"]
         if not self.override_extra:
             extras_template = (
                 EXTRA_PROJECT_CMD_TEMPLATE
@@ -597,8 +729,10 @@ class SubmissionConductor(object):
             _LOGGER.debug(f"namespace pipelines: { pl_iface }")
 
             namespaces["pipeline"]["var_templates"] = pl_iface[VAR_TEMPL_KEY] or {}
-            for k, v in namespaces["pipeline"]["var_templates"].items():
-                namespaces["pipeline"]["var_templates"][k] = expandpath(v)
+
+            namespaces["pipeline"]["var_templates"] = expand_nested_var_templates(
+                namespaces["pipeline"]["var_templates"], namespaces
+            )
 
             # pre_submit hook namespace updates
             namespaces = _exec_pre_submit(pl_iface, namespaces)
@@ -607,7 +741,6 @@ class SubmissionConductor(object):
                 argstring = jinja_render_template_strictly(
                     template=templ, namespaces=namespaces
                 )
-                print(argstring)
             except UndefinedError as jinja_exception:
                 _LOGGER.warning(NOT_SUB_MSG.format(str(jinja_exception)))
             except KeyError as e:

looper/const.py

@@ -1,6 +1,7 @@
 """ Shared project constants """
 
 import os
+from enum import Enum
 
 __author__ = "Databio lab"
 __email__ = "nathan@code.databio.org"
@@ -92,6 +93,7 @@ __all__ = [
     "DEBUG_EIDO_VALIDATION",
     "LOOPER_GENERIC_OUTPUT_SCHEMA",
     "LOOPER_GENERIC_COUNT_LINES",
+    "PipelineLevel",
 ]
 
 FLAGS = ["completed", "running", "failed", "waiting", "partial"]
@@ -268,3 +270,10 @@ MESSAGE_BY_SUBCOMMAND = {
     "init-piface": "Initialize generic pipeline interface.",
     "link": "Create directory of symlinks for reported results.",
 }
+
+# Add project/sample enum
+
+
+class PipelineLevel(Enum):
+    SAMPLE = "sample"
+    PROJECT = "project"

looper/divvy.py

@@ -1,16 +1,14 @@
 """ Computing configuration representation """
 
 import logging
-import logmuse
 import os
-import sys
 import shutil
-import yaml
-from yaml import SafeLoader
+
+
 from shutil import copytree
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+from yacman import write_lock, FILEPATH_KEY, load_yaml, select_config
 
-from ubiquerg import is_writable, VersionInHelpParser
-import yacman
 
 from .const import (
     COMPUTE_SETTINGS_VARNAME,
@@ -21,14 +19,13 @@ from .const import (
 )
 from .utils import write_submit_script
 
-# from . import __version__
 
 _LOGGER = logging.getLogger(__name__)
 
 # This is the divvy.py submodule from divvy
 
 
-class ComputingConfiguration(yacman.YAMLConfigManager):
+class ComputingConfiguration(YAMLConfigManager):
     """
     Represents computing configuration objects.
 
@@ -44,36 +41,31 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         `DIVCFG` file)
     """
 
-    def __init__(self, entries=None, filepath=None):
-        if not entries and not filepath:
-            # Handle the case of an empty one, when we'll use the default
-            filepath = select_divvy_config(None)
-
-        super(ComputingConfiguration, self).__init__(
-            entries=entries,
-            filepath=filepath,
-            schema_source=DEFAULT_CONFIG_SCHEMA,
-            validate_on_write=True,
+    def __init__(
+        self,
+        entries=None,
+        wait_max=None,
+        strict_ro_locks=False,
+        schema_source=None,
+        validate_on_write=False,
+    ):
+        super().__init__(
+            entries, wait_max, strict_ro_locks, schema_source, validate_on_write
         )
 
-        if not "compute_packages" in self:
-            raise Exception(
-                "Your divvy config file is not in divvy config format "
-                "(it lacks a compute_packages section): '{}'".format(filepath)
-            )
-            # We require that compute_packages be present, even if empty
+        if "compute_packages" not in self:
             self["compute_packages"] = {}
-
         # Initialize default compute settings.
         _LOGGER.debug("Establishing project compute settings")
         self.compute = None
         self.setdefault("adapters", None)
         self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
-        self.config_file = self.filepath
 
     def write(self, filename=None):
-        super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
-        filename = filename or getattr(self, yacman.FILEPATH_KEY)
+        with write_lock(self) as locked_ym:
+            locked_ym.rebase()
+            locked_ym.write()
+        filename = filename or getattr(self, FILEPATH_KEY)
         filedir = os.path.dirname(filename)
         # For this object, we *also* have to write the template files
         for pkg_name, pkg in self["compute_packages"].items():
@@ -119,9 +111,12 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
 
         :return str: path to folder with default submission templates
         """
-        return os.path.join(
-            os.path.dirname(__file__), "default_config", "divvy_templates"
-        )
+        if self.filepath:
+            return os.path.join(os.path.dirname(self.filepath), "divvy_templates")
+        else:
+            return os.path.join(
+                os.path.dirname(__file__), "default_config", "divvy_templates"
+            )
 
     def activate_package(self, package_name):
         """
@@ -151,23 +146,30 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
             # Augment compute, creating it if needed.
             if self.compute is None:
                 _LOGGER.debug("Creating Project compute")
-                self.compute = yacman.YAMLConfigManager()
+                self.compute = YAMLConfigManager()
                 _LOGGER.debug(
                     "Adding entries for package_name '{}'".format(package_name)
                 )
 
-            self.compute.update(self["compute_packages"][package_name])
+            self.compute.update_from_obj(self["compute_packages"][package_name])
 
             # Ensure submission template is absolute. This *used to be* handled
             # at update (so the paths were stored as absolutes in the packages),
             # but now, it makes more sense to do it here so we can piggyback on
             # the default update() method and not even have to do that.
             if not os.path.isabs(self.compute["submission_template"]):
+
                 try:
-                    self.compute["submission_template"] = os.path.join(
-                        os.path.dirname(self.filepath),
-                        self.compute["submission_template"],
-                    )
+                    if self.filepath:
+                        self.compute["submission_template"] = os.path.join(
+                            os.path.dirname(self.filepath),
+                            self.compute["submission_template"],
+                        )
+                    else:
+                        self.compute["submission_template"] = os.path.join(
+                            os.path.dirname(self.default_config_file),
+                            self.compute["submission_template"],
+                        )
                 except AttributeError as e:
                     # Environment and environment compute should at least have been
                     # set as null-valued attributes, so execution here is an error.
@@ -200,14 +202,19 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         self.reset_active_settings()
         return self.activate_package(package_name)
 
-    def get_active_package(self):
+    def get_active_package(self) -> YAMLConfigManager:
         """
         Returns settings for the currently active compute package
 
-        :return yacman.YacAttMap: data defining the active compute package
+        :return YAMLConfigManager: data defining the active compute package
         """
         return self.compute
 
+    @property
+    def compute_packages(self):
+
+        return self["compute_packages"]
+
     def list_compute_packages(self):
         """
         Returns a list of available compute packages.
@@ -222,7 +229,7 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
 
         :return bool: success flag
         """
-        self.compute = yacman.YacAttMap()
+        self.compute = YAMLConfigManager()
         return True
 
     def update_packages(self, config_file):
@@ -235,11 +242,11 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
 
         :param str config_file: path to file with new divvy configuration data
         """
-        entries = yacman.load_yaml(config_file)
+        entries = load_yaml(config_file)
         self.update(entries)
         return True
 
-    def get_adapters(self):
+    def get_adapters(self) -> YAMLConfigManager:
         """
         Get current adapters, if defined.
 
@@ -248,9 +255,9 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         package-specific set of adapters, if any defined in 'adapters' section
         under currently active compute package.
 
-        :return yacman.YAMLConfigManager: current adapters mapping
+        :return YAMLConfigManager: current adapters mapping
         """
-        adapters = yacman.YAMLConfigManager()
+        adapters = YAMLConfigManager()
         if "adapters" in self and self["adapters"] is not None:
             adapters.update(self["adapters"])
         if "compute" in self and "adapters" in self.compute:
@@ -376,7 +383,7 @@ def select_divvy_config(filepath):
     :param str | NoneType filepath: direct file path specification
     :return str: path to the config file to read
     """
-    divcfg = yacman.select_config(
+    divcfg = select_config(
         config_filepath=filepath,
         config_env_vars=COMPUTE_SETTINGS_VARNAME,
         default_config_filepath=DEFAULT_CONFIG_FILEPATH,
@@ -404,11 +411,13 @@ def divvy_init(config_path, template_config_path):
         _LOGGER.error("You must specify a template config file path.")
         return
 
+    if not os.path.isabs(config_path):
+        config_path = os.path.abspath(config_path)
+
     if config_path and not os.path.exists(config_path):
-        # dcc.write(config_path)
         # Init should *also* write the templates.
         dest_folder = os.path.dirname(config_path)
-        copytree(os.path.dirname(template_config_path), dest_folder)
+        copytree(os.path.dirname(template_config_path), dest_folder, dirs_exist_ok=True)
         template_subfolder = os.path.join(dest_folder, "divvy_templates")
         _LOGGER.info("Wrote divvy templates to folder: {}".format(template_subfolder))
         new_template = os.path.join(

looper/exceptions.py

@@ -15,6 +15,7 @@ _all__ = [
     "PipelineInterfaceConfigError",
     "PipelineInterfaceRequirementsError",
     "MisconfigurationException",
+    "LooperReportError",
 ]
 
 
@@ -31,7 +32,7 @@ class SampleFailedException(LooperError):
 
 
 class MisconfigurationException(LooperError):
-    """Duplication of pipeline identifier precludes unique pipeline ref."""
+    """Looper not properly configured"""
 
     def __init__(self, key):
         super(MisconfigurationException, self).__init__(key)
@@ -109,3 +110,10 @@ class PipelineInterfaceRequirementsError(LooperError):
             )
         )
         self.error_specs = typename_by_requirement
+
+
+class LooperReportError(LooperError):
+    """Looper reporting errors"""
+
+    def __init__(self, reason):
+        super(LooperReportError, self).__init__(reason)