looper 1.7.0a1__py3-none-any.whl → 1.7.1__py3-none-any.whl

looper/_version.py

@@ -1 +1 @@
-__version__ = "1.7.0a1"
+__version__ = "1.7.1"

looper/cli_looper.py

@@ -219,19 +219,27 @@ def build_parser():
         for subparser in [run_subparser, rerun_subparser]:
             subparser.add_argument(
                 "-u",
-                "--lump",
+                "--lump-s",
                 default=None,
                 metavar="X",
                 type=html_range(min_val=0, max_val=100, step=0.1, value=0),
-                help="Total input file size (GB) to batch into one job",
+                help="Lump by size: total input file size (GB) to batch into one job",
             )
             subparser.add_argument(
                 "-n",
-                "--lumpn",
+                "--lump-n",
                 default=None,
                 metavar="N",
                 type=html_range(min_val=1, max_val="num_samples", value=1),
-                help="Number of commands to batch into one job",
+                help="Lump by number: number of samples to batch into one job",
+            )
+            subparser.add_argument(
+                "-j",
+                "--lump-j",
+                default=None,
+                metavar="J",
+                type=int,
+                help="Lump samples into number of jobs.",
             )
 
         check_subparser.add_argument(

looper/conductor.py

@@ -6,6 +6,7 @@ import os
 import subprocess
 import time
 import yaml
+from math import ceil
 from copy import copy, deepcopy
 from json import loads
 from subprocess import check_output
@@ -132,6 +133,7 @@ class SubmissionConductor(object):
         compute_variables=None,
         max_cmds=None,
         max_size=None,
+        max_jobs=None,
         automatic=True,
         collate=False,
     ):
@@ -166,6 +168,8 @@ class SubmissionConductor(object):
             include in a single job script.
         :param int | float | NoneType max_size: Upper bound on total file
             size of inputs used by the commands lumped into single job script.
+        :param int | float | NoneType max_jobs: Upper bound on total number of jobs to
+            group samples for submission.
         :param bool automatic: Whether the submission should be automatic once
             the pool reaches capacity.
         :param bool collate: Whether a collate job is to be submitted (runs on
@@ -200,6 +204,16 @@ class SubmissionConductor(object):
                 "{}".format(self.extra_pipe_args)
             )
 
+        if max_jobs:
+            if max_jobs == 0 or max_jobs < 0:
+                raise ValueError(
+                    "If specified, max job command count must be a positive integer, greater than zero."
+                )
+
+            num_samples = len(self.prj.samples)
+            samples_per_job = num_samples / max_jobs
+            max_cmds = ceil(samples_per_job)
+
         if not self.collate:
             self.automatic = automatic
             if max_cmds is None and max_size is None:

looper/looper.py

@@ -404,8 +404,9 @@ class Runner(Executor):
                 extra_args=args.command_extra,
                 extra_args_override=args.command_extra_override,
                 ignore_flags=args.ignore_flags,
-                max_cmds=args.lumpn,
-                max_size=args.lump,
+                max_cmds=args.lump_n,
+                max_size=args.lump_s,
+                max_jobs=args.lump_j,
             )
             submission_conductors[piface.pipe_iface_file] = conductor
 

{looper-1.7.0a1.dist-info → looper-1.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: looper
-Version: 1.7.0a1
+Version: 1.7.1
 Summary: A pipeline submission engine that parses sample inputs and submits pipelines for each sample.
 Home-page: https://github.com/pepkit/looper
 Author: Nathan Sheffield, Vince Reuter, Michal Stolarczyk, Johanna Klughammer, Andre Rendeiro
@@ -23,7 +23,7 @@ Requires-Dist: logmuse >=0.2.0
 Requires-Dist: pandas >=2.0.2
 Requires-Dist: pephubclient >=0.1.2
 Requires-Dist: peppy >=0.40.0
-Requires-Dist: pipestat >=0.7.1a1
+Requires-Dist: pipestat <0.9.0,>=0.8.0
 Requires-Dist: pyyaml >=3.12
 Requires-Dist: rich >=9.10.0
 Requires-Dist: ubiquerg >=0.5.2

{looper-1.7.0a1.dist-info → looper-1.7.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 looper/__init__.py,sha256=f_z9YY4ibOk7eyWoaViH_VaCXMlPQeiftbnibSFj-3E,1333
 looper/__main__.py,sha256=8CX2ae8mUQNI_Z8pdBT4i5UFqROFX1awyFnuYCKuYXg,238
-looper/_version.py,sha256=oLP-5l-VhQk8ffqYJg1za2BCPImuvh36kUpvrjML1zg,24
+looper/_version.py,sha256=rHqoOqa3LxWXvrBmnnm2LDmV9IlmMEb5qOmw5doj3fk,22
 looper/cli_divvy.py,sha256=J07x83sqC4jJeu3_yS6KOARPWmwKGAV7JvN33T5zDac,5907
-looper/cli_looper.py,sha256=AQZAmycnhKyoPbPN1zoWfKlBm-EOSyyoov-8L7DoIQQ,26092
-looper/conductor.py,sha256=BmMATwtkHoPSsffYopCQ1WxK83O3UjOTFRrakgD6DzA,30231
+looper/cli_looper.py,sha256=se-EbQ4nucWxMiU0VLnmV0Kss-JMjSmWih6vHaOiLi0,26367
+looper/conductor.py,sha256=9k0r_vsHCP25MrDjz7GR16_EEdqP9JZELYZJVm6ny1g,30777
 looper/const.py,sha256=bPj4lTuj2l6gwHROWqj16iHfJFo9ghZAz8THNREWW4U,8558
 looper/divvy.py,sha256=qa1ebbQTfNupAyDfhfEJ6mbZ_V3zk-D_E-Tck7miJ38,15688
 looper/exceptions.py,sha256=r6SKKt-m8CXQnXGDnuiwoA6zBJhIZflygBKjX4RCloI,3419
-looper/looper.py,sha256=WekyR0K__G-ZMEDYUyWCBP6MDBah_ykCv6cfGMztFqc,30407
+looper/looper.py,sha256=U3mNqXRDZRzmFXV26TcPQYKPdY1plPvMraNHTnps4mQ,30448
 looper/parser_types.py,sha256=d3FHt54f9jo9VZMr5SQkbghcAdABqiYZW2JBGO5EBnw,2327
 looper/pipeline_interface.py,sha256=y46tB1_73d1FX8N1w4-GGvRBJ7rqhenuUYVtUfIhK5s,14974
 looper/plugins.py,sha256=MaMdPmK9U_4FkNJE5kccohBbY1i2qj1NTEucubFOJek,5747
@@ -55,9 +55,9 @@ looper/schemas/divvy_config_schema.yaml,sha256=7GJfKLc3VX4RGjHnOE1zxwsHXhj_ur9za
 looper/schemas/pipeline_interface_schema_generic.yaml,sha256=D16Rkpj03H9WnvA_N18iNU-hH_HwOuyESJ8Hk5hZSXc,1518
 looper/schemas/pipeline_interface_schema_project.yaml,sha256=-ZWyA0lKXWik3obuLNVk3IsAZYfbLVbCDvJnD-Fcluo,1567
 looper/schemas/pipeline_interface_schema_sample.yaml,sha256=x0OwVnijJpvm50DscvvJujdK4UAI7d71pqVemQS-D-0,1564
-looper-1.7.0a1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
-looper-1.7.0a1.dist-info/METADATA,sha256=5TYw8-UT-e8bstmvw6WM3M6ayupc5V2DP0dPf4qX-uI,1737
-looper-1.7.0a1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-looper-1.7.0a1.dist-info/entry_points.txt,sha256=AEL1eb0gPLYvAEUewM35Ng4scXGZIWJK4Mxdj3Hm8Fw,83
-looper-1.7.0a1.dist-info/top_level.txt,sha256=I0Yf7djsoQAMzwHBbDiQi9hGtq4Z41_Ma5CX8qXG8Y8,7
-looper-1.7.0a1.dist-info/RECORD,,
+looper-1.7.1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
+looper-1.7.1.dist-info/METADATA,sha256=RRHlAK9WkJv1b9KtkZpMXHk2xyATbe5DJWCcRZMq7Wk,1740
+looper-1.7.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+looper-1.7.1.dist-info/entry_points.txt,sha256=AEL1eb0gPLYvAEUewM35Ng4scXGZIWJK4Mxdj3Hm8Fw,83
+looper-1.7.1.dist-info/top_level.txt,sha256=I0Yf7djsoQAMzwHBbDiQi9hGtq4Z41_Ma5CX8qXG8Y8,7
+looper-1.7.1.dist-info/RECORD,,

{looper-1.7.0a1.dist-info → looper-1.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any