looper 1.7.0a1__py3-none-any.whl → 1.8.0__py3-none-any.whl

looper/conductor.py

@@ -6,6 +6,7 @@ import os
 import subprocess
 import time
 import yaml
+from math import ceil
 from copy import copy, deepcopy
 from json import loads
 from subprocess import check_output
@@ -20,12 +21,13 @@ from peppy.exceptions import RemoteYAMLError
 from pipestat import PipestatError
 from ubiquerg import expandpath, is_command_callable
 from yaml import dump
-from yacman import YAMLConfigManager
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
 
 from .const import *
 from .exceptions import JobSubmissionException, SampleFailedException
 from .processed_project import populate_sample_paths
 from .utils import fetch_sample_flags, jinja_render_template_strictly
+from .const import PipelineLevel
 
 
 _LOGGER = logging.getLogger(__name__)
@@ -84,11 +86,23 @@ def _get_yaml_path(namespaces, template_key, default_name_appendix="", filename=
 
 def write_pipestat_config(looper_pipestat_config_path, pipestat_config_dict):
     """
-    This is run at the project level, not at the sample level.
+    This writes a combined configuration file to be passed to a PipestatManager.
+    :param str looper_pipestat_config_path: path to the created pipestat configuration file
+    :param dict pipestat_config_dict: the dict containing key value pairs to be written to the pipestat configutation
+    return bool
     """
+
+    if not os.path.exists(os.path.dirname(looper_pipestat_config_path)):
+        try:
+            os.makedirs(os.path.dirname(looper_pipestat_config_path))
+        except FileExistsError:
+            pass
+
     with open(looper_pipestat_config_path, "w") as f:
         yaml.dump(pipestat_config_dict, f)
-    print(f"Initialized looper config file: {looper_pipestat_config_path}")
+    _LOGGER.debug(
+        msg=f"Initialized pipestat config file: {looper_pipestat_config_path}"
+    )
 
     return True
 
@@ -132,6 +146,7 @@ class SubmissionConductor(object):
         compute_variables=None,
         max_cmds=None,
         max_size=None,
+        max_jobs=None,
         automatic=True,
         collate=False,
     ):
@@ -166,6 +181,8 @@ class SubmissionConductor(object):
             include in a single job script.
         :param int | float | NoneType max_size: Upper bound on total file
             size of inputs used by the commands lumped into single job script.
+        :param int | float | NoneType max_jobs: Upper bound on total number of jobs to
+            group samples for submission.
         :param bool automatic: Whether the submission should be automatic once
             the pool reaches capacity.
         :param bool collate: Whether a collate job is to be submitted (runs on
@@ -200,6 +217,16 @@ class SubmissionConductor(object):
                 "{}".format(self.extra_pipe_args)
             )
 
+        if max_jobs:
+            if max_jobs == 0 or max_jobs < 0:
+                raise ValueError(
+                    "If specified, max job command count must be a positive integer, greater than zero."
+                )
+
+            num_samples = len(self.prj.samples)
+            samples_per_job = num_samples / max_jobs
+            max_cmds = ceil(samples_per_job)
+
         if not self.collate:
             self.automatic = automatic
             if max_cmds is None and max_size is None:
@@ -247,8 +274,12 @@ class SubmissionConductor(object):
 
         :param bool frorce: whether to force the project submission (ignore status/flags)
         """
+        psms = {}
         if self.prj.pipestat_configured_project:
-            psm = self.prj.get_pipestat_managers(project_level=True)[self.pl_name]
+            for piface in self.prj.project_pipeline_interfaces:
+                if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
+                    psms[piface.psm.pipeline_name] = piface.psm
+            psm = psms[self.pl_name]
             status = psm.get_status()
             if not force and status is not None:
                 _LOGGER.info(f"> Skipping project. Determined status: {status}")
@@ -274,12 +305,11 @@ class SubmissionConductor(object):
             )
         )
         if self.prj.pipestat_configured:
-            psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
-            sample_statuses = psms[self.pl_name].get_status(
+            sample_statuses = self.pl_iface.psm.get_status(
                 record_identifier=sample.sample_name
             )
             if sample_statuses == "failed" and rerun is True:
-                psms[self.pl_name].set_status(
+                self.pl_iface.psm.set_status(
                     record_identifier=sample.sample_name, status_identifier="waiting"
                 )
                 sample_statuses = "waiting"
@@ -289,23 +319,27 @@ class SubmissionConductor(object):
 
         use_this_sample = True  # default to running this sample
         msg = None
+        if rerun and sample_statuses == []:
+            msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found."
+            use_this_sample = False
         if sample_statuses:
             status_str = ", ".join(sample_statuses)
             failed_flag = any("failed" in x for x in sample_statuses)
+            waiting_flag = any("waiting" in x for x in sample_statuses)
             if self.ignore_flags:
                 msg = f"> Found existing status: {status_str}. Ignoring."
             else:  # this pipeline already has a status
                 msg = f"> Found existing status: {status_str}. Skipping sample."
-                if failed_flag:
+                if failed_flag and not rerun:
                     msg += " Use rerun to ignore failed status."  # help guidance
                 use_this_sample = False
             if rerun:
                 # Rescue the sample if rerun requested, and failed flag is found
-                if failed_flag:
-                    msg = f"> Re-running failed sample. Status: {status_str}"
+                if failed_flag or waiting_flag:
+                    msg = f"> Re-running sample. Status: {status_str}"
                     use_this_sample = True
                 else:
-                    msg = f"> Skipping sample because rerun requested, but no failed flag found. Status: {status_str}"
+                    msg = f"> Skipping sample because rerun requested, but no failed or waiting flag found. Status: {status_str}"
                     use_this_sample = False
         if msg:
             _LOGGER.info(msg)
@@ -514,12 +548,7 @@ class SubmissionConductor(object):
         :return yacman.YAMLConfigManager: pipestat namespace
         """
         try:
-            psms = (
-                self.prj.get_pipestat_managers(sample_name)
-                if sample_name
-                else self.prj.get_pipestat_managers(project_level=True)
-            )
-            psm = psms[self.pl_iface.pipeline_name]
+            psm = self.pl_iface.psm
         except (PipestatError, AttributeError) as e:
             # pipestat section faulty or not found in project.looper or sample
             # or project is missing required pipestat attributes
@@ -607,7 +636,6 @@ class SubmissionConductor(object):
                 argstring = jinja_render_template_strictly(
                     template=templ, namespaces=namespaces
                 )
-                print(argstring)
             except UndefinedError as jinja_exception:
                 _LOGGER.warning(NOT_SUB_MSG.format(str(jinja_exception)))
             except KeyError as e:

looper/const.py

@@ -1,6 +1,7 @@
 """ Shared project constants """
 
 import os
+from enum import Enum
 
 __author__ = "Databio lab"
 __email__ = "nathan@code.databio.org"
@@ -268,3 +269,10 @@ MESSAGE_BY_SUBCOMMAND = {
     "init-piface": "Initialize generic pipeline interface.",
     "link": "Create directory of symlinks for reported results.",
 }
+
+# Add project/sample enum
+
+
+class PipelineLevel(Enum):
+    SAMPLE = "sample"
+    PROJECT = "project"

looper/divvy.py

@@ -6,11 +6,14 @@ import os
 import sys
 import shutil
 import yaml
-from yaml import SafeLoader
-from shutil import copytree
 
+
+from shutil import copytree
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+from yacman import write_lock, FILEPATH_KEY, load_yaml, select_config
+from yaml import SafeLoader
 from ubiquerg import is_writable, VersionInHelpParser
-import yacman
+
 
 from .const import (
     COMPUTE_SETTINGS_VARNAME,
@@ -28,7 +31,7 @@ _LOGGER = logging.getLogger(__name__)
 # This is the divvy.py submodule from divvy
 
 
-class ComputingConfiguration(yacman.YAMLConfigManager):
+class ComputingConfiguration(YAMLConfigManager):
     """
     Represents computing configuration objects.
 
@@ -44,36 +47,31 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         `DIVCFG` file)
     """
 
-    def __init__(self, entries=None, filepath=None):
-        if not entries and not filepath:
-            # Handle the case of an empty one, when we'll use the default
-            filepath = select_divvy_config(None)
-
-        super(ComputingConfiguration, self).__init__(
-            entries=entries,
-            filepath=filepath,
-            schema_source=DEFAULT_CONFIG_SCHEMA,
-            validate_on_write=True,
+    def __init__(
+        self,
+        entries=None,
+        wait_max=None,
+        strict_ro_locks=False,
+        schema_source=None,
+        validate_on_write=False,
+    ):
+        super().__init__(
+            entries, wait_max, strict_ro_locks, schema_source, validate_on_write
         )
 
-        if not "compute_packages" in self:
-            raise Exception(
-                "Your divvy config file is not in divvy config format "
-                "(it lacks a compute_packages section): '{}'".format(filepath)
-            )
-            # We require that compute_packages be present, even if empty
+        if "compute_packages" not in self:
             self["compute_packages"] = {}
-
         # Initialize default compute settings.
         _LOGGER.debug("Establishing project compute settings")
         self.compute = None
         self.setdefault("adapters", None)
         self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
-        self.config_file = self.filepath
 
     def write(self, filename=None):
-        super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
-        filename = filename or getattr(self, yacman.FILEPATH_KEY)
+        with write_lock(self) as locked_ym:
+            locked_ym.rebase()
+            locked_ym.write()
+        filename = filename or getattr(self, FILEPATH_KEY)
         filedir = os.path.dirname(filename)
         # For this object, we *also* have to write the template files
         for pkg_name, pkg in self["compute_packages"].items():
@@ -151,12 +149,12 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
             # Augment compute, creating it if needed.
             if self.compute is None:
                 _LOGGER.debug("Creating Project compute")
-                self.compute = yacman.YAMLConfigManager()
+                self.compute = YAMLConfigManager()
                 _LOGGER.debug(
                     "Adding entries for package_name '{}'".format(package_name)
                 )
 
-            self.compute.update(self["compute_packages"][package_name])
+            self.compute.update_from_obj(self["compute_packages"][package_name])
 
             # Ensure submission template is absolute. This *used to be* handled
             # at update (so the paths were stored as absolutes in the packages),
@@ -165,7 +163,7 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
             if not os.path.isabs(self.compute["submission_template"]):
                 try:
                     self.compute["submission_template"] = os.path.join(
-                        os.path.dirname(self.filepath),
+                        os.path.dirname(self.default_config_file),
                         self.compute["submission_template"],
                     )
                 except AttributeError as e:
@@ -200,11 +198,11 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         self.reset_active_settings()
         return self.activate_package(package_name)
 
-    def get_active_package(self):
+    def get_active_package(self) -> YAMLConfigManager:
         """
         Returns settings for the currently active compute package
 
-        :return yacman.YacAttMap: data defining the active compute package
+        :return YAMLConfigManager: data defining the active compute package
         """
         return self.compute
 
@@ -222,7 +220,7 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
 
         :return bool: success flag
         """
-        self.compute = yacman.YacAttMap()
+        self.compute = YAMLConfigManager()
         return True
 
     def update_packages(self, config_file):
@@ -235,11 +233,11 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
 
         :param str config_file: path to file with new divvy configuration data
         """
-        entries = yacman.load_yaml(config_file)
+        entries = load_yaml(config_file)
         self.update(entries)
         return True
 
-    def get_adapters(self):
+    def get_adapters(self) -> YAMLConfigManager:
         """
         Get current adapters, if defined.
 
@@ -248,9 +246,9 @@ class ComputingConfiguration(yacman.YAMLConfigManager):
         package-specific set of adapters, if any defined in 'adapters' section
         under currently active compute package.
 
-        :return yacman.YAMLConfigManager: current adapters mapping
+        :return YAMLConfigManager: current adapters mapping
         """
-        adapters = yacman.YAMLConfigManager()
+        adapters = YAMLConfigManager()
         if "adapters" in self and self["adapters"] is not None:
             adapters.update(self["adapters"])
         if "compute" in self and "adapters" in self.compute:
@@ -376,7 +374,7 @@ def select_divvy_config(filepath):
     :param str | NoneType filepath: direct file path specification
     :return str: path to the config file to read
     """
-    divcfg = yacman.select_config(
+    divcfg = select_config(
         config_filepath=filepath,
         config_env_vars=COMPUTE_SETTINGS_VARNAME,
         default_config_filepath=DEFAULT_CONFIG_FILEPATH,