looper 1.5.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/__init__.py +3 -498
- looper/__main__.py +2 -2
- looper/_version.py +1 -1
- looper/cli_divvy.py +182 -0
- looper/cli_looper.py +776 -0
- looper/conductor.py +53 -206
- looper/const.py +51 -3
- looper/divvy.py +28 -196
- looper/exceptions.py +18 -0
- looper/looper.py +177 -612
- looper/plugins.py +160 -0
- looper/processed_project.py +1 -1
- looper/project.py +229 -117
- looper/utils.py +119 -43
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/METADATA +6 -6
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/RECORD +20 -20
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/WHEEL +1 -1
- looper/html_reports.py +0 -1057
- looper/html_reports_pipestat.py +0 -924
- looper/html_reports_project_pipestat.py +0 -269
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/LICENSE.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/entry_points.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/top_level.txt +0 -0
looper/divvy.py
CHANGED
@@ -28,7 +28,7 @@ _LOGGER = logging.getLogger(__name__)
|
|
28
28
|
# This is the divvy.py submodule from divvy
|
29
29
|
|
30
30
|
|
31
|
-
class ComputingConfiguration(yacman.
|
31
|
+
class ComputingConfiguration(yacman.YAMLConfigManager):
|
32
32
|
"""
|
33
33
|
Represents computing configuration objects.
|
34
34
|
|
@@ -53,30 +53,30 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
53
53
|
entries=entries,
|
54
54
|
filepath=filepath,
|
55
55
|
schema_source=DEFAULT_CONFIG_SCHEMA,
|
56
|
-
|
56
|
+
validate_on_write=True,
|
57
57
|
)
|
58
58
|
|
59
|
-
if not
|
59
|
+
if not "compute_packages" in self:
|
60
60
|
raise Exception(
|
61
61
|
"Your divvy config file is not in divvy config format "
|
62
62
|
"(it lacks a compute_packages section): '{}'".format(filepath)
|
63
63
|
)
|
64
64
|
# We require that compute_packages be present, even if empty
|
65
|
-
self
|
65
|
+
self["compute_packages"] = {}
|
66
66
|
|
67
67
|
# Initialize default compute settings.
|
68
68
|
_LOGGER.debug("Establishing project compute settings")
|
69
69
|
self.compute = None
|
70
70
|
self.setdefault("adapters", None)
|
71
71
|
self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
|
72
|
-
self.config_file = self
|
72
|
+
self.config_file = self.filepath
|
73
73
|
|
74
74
|
def write(self, filename=None):
|
75
75
|
super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
|
76
76
|
filename = filename or getattr(self, yacman.FILEPATH_KEY)
|
77
77
|
filedir = os.path.dirname(filename)
|
78
78
|
# For this object, we *also* have to write the template files
|
79
|
-
for pkg_name, pkg in self
|
79
|
+
for pkg_name, pkg in self["compute_packages"].items():
|
80
80
|
print(pkg)
|
81
81
|
destfile = os.path.join(filedir, os.path.basename(pkg.submission_template))
|
82
82
|
shutil.copyfile(pkg.submission_template, destfile)
|
@@ -109,7 +109,7 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
109
109
|
|
110
110
|
:return str: submission script content template for current state
|
111
111
|
"""
|
112
|
-
with open(self.compute
|
112
|
+
with open(self.compute["submission_template"], "r") as f:
|
113
113
|
return f.read()
|
114
114
|
|
115
115
|
@property
|
@@ -145,28 +145,28 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
145
145
|
|
146
146
|
if (
|
147
147
|
package_name
|
148
|
-
and self
|
149
|
-
and package_name in self
|
148
|
+
and self["compute_packages"]
|
149
|
+
and package_name in self["compute_packages"]
|
150
150
|
):
|
151
151
|
# Augment compute, creating it if needed.
|
152
152
|
if self.compute is None:
|
153
153
|
_LOGGER.debug("Creating Project compute")
|
154
|
-
self.compute = yacman.
|
154
|
+
self.compute = yacman.YAMLConfigManager()
|
155
155
|
_LOGGER.debug(
|
156
156
|
"Adding entries for package_name '{}'".format(package_name)
|
157
157
|
)
|
158
158
|
|
159
|
-
self.compute.
|
159
|
+
self.compute.update(self["compute_packages"][package_name])
|
160
160
|
|
161
161
|
# Ensure submission template is absolute. This *used to be* handled
|
162
162
|
# at update (so the paths were stored as absolutes in the packages),
|
163
163
|
# but now, it makes more sense to do it here so we can piggyback on
|
164
164
|
# the default update() method and not even have to do that.
|
165
|
-
if not os.path.isabs(self.compute
|
165
|
+
if not os.path.isabs(self.compute["submission_template"]):
|
166
166
|
try:
|
167
|
-
self.compute
|
168
|
-
os.path.dirname(self
|
169
|
-
self.compute
|
167
|
+
self.compute["submission_template"] = os.path.join(
|
168
|
+
os.path.dirname(self.filepath),
|
169
|
+
self.compute["submission_template"],
|
170
170
|
)
|
171
171
|
except AttributeError as e:
|
172
172
|
# Environment and environment compute should at least have been
|
@@ -174,7 +174,7 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
174
174
|
_LOGGER.error(str(e))
|
175
175
|
|
176
176
|
_LOGGER.debug(
|
177
|
-
"Submit template set to: {}".format(self.compute
|
177
|
+
"Submit template set to: {}".format(self.compute["submission_template"])
|
178
178
|
)
|
179
179
|
|
180
180
|
return True
|
@@ -184,7 +184,7 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
184
184
|
# both present--but don't evaluate to True--is fairly harmless.
|
185
185
|
_LOGGER.debug(
|
186
186
|
"Can't activate package. compute_packages = {}".format(
|
187
|
-
self
|
187
|
+
self["compute_packages"]
|
188
188
|
)
|
189
189
|
)
|
190
190
|
|
@@ -214,7 +214,7 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
214
214
|
|
215
215
|
:return set[str]: names of available compute packages
|
216
216
|
"""
|
217
|
-
return set(self
|
217
|
+
return set(self["compute_packages"].keys())
|
218
218
|
|
219
219
|
def reset_active_settings(self):
|
220
220
|
"""
|
@@ -248,13 +248,13 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
248
248
|
package-specific set of adapters, if any defined in 'adapters' section
|
249
249
|
under currently active compute package.
|
250
250
|
|
251
|
-
:return yacman.
|
251
|
+
:return yacman.YAMLConfigManager: current adapters mapping
|
252
252
|
"""
|
253
|
-
adapters = yacman.
|
254
|
-
if "adapters" in self and self
|
255
|
-
adapters.update(self
|
253
|
+
adapters = yacman.YAMLConfigManager()
|
254
|
+
if "adapters" in self and self["adapters"] is not None:
|
255
|
+
adapters.update(self["adapters"])
|
256
256
|
if "compute" in self and "adapters" in self.compute:
|
257
|
-
adapters.update(self.compute
|
257
|
+
adapters.update(self.compute["adapters"])
|
258
258
|
if not adapters:
|
259
259
|
_LOGGER.debug("No adapters determined in divvy configuration file.")
|
260
260
|
return adapters
|
@@ -270,7 +270,9 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
270
270
|
self.submit(temp.name, extra_vars)
|
271
271
|
else:
|
272
272
|
script = self.write_script(output_path, extra_vars)
|
273
|
-
submission_command = "{} {}".format(
|
273
|
+
submission_command = "{} {}".format(
|
274
|
+
self.compute["submission_command"], script
|
275
|
+
)
|
274
276
|
_LOGGER.info(submission_command)
|
275
277
|
os.system(submission_command)
|
276
278
|
|
@@ -337,7 +339,7 @@ class ComputingConfiguration(yacman.YacAttMap):
|
|
337
339
|
if len(extra_var) > 0 and list(extra_var.keys())[0] not in exclude:
|
338
340
|
variables.update(extra_var)
|
339
341
|
_LOGGER.debug(
|
340
|
-
"Submission template: {}".format(self.compute
|
342
|
+
"Submission template: {}".format(self.compute["submission_template"])
|
341
343
|
)
|
342
344
|
if output_path:
|
343
345
|
_LOGGER.info("Writing script to {}".format(os.path.abspath(output_path)))
|
@@ -379,6 +381,7 @@ def select_divvy_config(filepath):
|
|
379
381
|
config_env_vars=COMPUTE_SETTINGS_VARNAME,
|
380
382
|
default_config_filepath=DEFAULT_CONFIG_FILEPATH,
|
381
383
|
check_exist=True,
|
384
|
+
config_name="divvy",
|
382
385
|
)
|
383
386
|
_LOGGER.debug("Selected divvy config: {}".format(divcfg))
|
384
387
|
return divcfg
|
@@ -415,174 +418,3 @@ def divvy_init(config_path, template_config_path):
|
|
415
418
|
_LOGGER.info("Wrote new divvy configuration file: {}".format(config_path))
|
416
419
|
else:
|
417
420
|
_LOGGER.warning("Can't initialize, file exists: {} ".format(config_path))
|
418
|
-
|
419
|
-
|
420
|
-
def build_argparser():
|
421
|
-
"""
|
422
|
-
Builds argument parser.
|
423
|
-
|
424
|
-
:return argparse.ArgumentParser
|
425
|
-
"""
|
426
|
-
|
427
|
-
banner = (
|
428
|
-
"%(prog)s - write compute job scripts that can be submitted to "
|
429
|
-
"any computing resource"
|
430
|
-
)
|
431
|
-
additional_description = "\nhttps://divvy.databio.org"
|
432
|
-
|
433
|
-
parser = VersionInHelpParser(
|
434
|
-
prog="divvy",
|
435
|
-
description=banner,
|
436
|
-
epilog=additional_description,
|
437
|
-
# version=__version__,
|
438
|
-
)
|
439
|
-
|
440
|
-
subparsers = parser.add_subparsers(dest="command")
|
441
|
-
|
442
|
-
def add_subparser(cmd, description):
|
443
|
-
return subparsers.add_parser(cmd, description=description, help=description)
|
444
|
-
|
445
|
-
subparser_messages = {
|
446
|
-
"init": "Initialize a new divvy config file",
|
447
|
-
"list": "List available compute packages",
|
448
|
-
"write": "Write a job script",
|
449
|
-
"submit": "Write and then submit a job script",
|
450
|
-
"inspect": "Inspect compute package",
|
451
|
-
}
|
452
|
-
|
453
|
-
sps = {}
|
454
|
-
for cmd, desc in subparser_messages.items():
|
455
|
-
sps[cmd] = add_subparser(cmd, desc)
|
456
|
-
# sps[cmd].add_argument(
|
457
|
-
# "config", nargs="?", default=None,
|
458
|
-
# help="Divvy configuration file.")
|
459
|
-
|
460
|
-
for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
|
461
|
-
sp.add_argument(
|
462
|
-
"config", nargs="?", default=None, help="Divvy configuration file."
|
463
|
-
)
|
464
|
-
|
465
|
-
sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
|
466
|
-
|
467
|
-
for sp in [sps["inspect"]]:
|
468
|
-
sp.add_argument(
|
469
|
-
"-p",
|
470
|
-
"--package",
|
471
|
-
default=DEFAULT_COMPUTE_RESOURCES_NAME,
|
472
|
-
help="Select from available compute packages",
|
473
|
-
)
|
474
|
-
|
475
|
-
for sp in [sps["write"], sps["submit"]]:
|
476
|
-
sp.add_argument(
|
477
|
-
"-s",
|
478
|
-
"--settings",
|
479
|
-
help="YAML file with job settings to populate the template",
|
480
|
-
)
|
481
|
-
|
482
|
-
sp.add_argument(
|
483
|
-
"-p",
|
484
|
-
"--package",
|
485
|
-
default=DEFAULT_COMPUTE_RESOURCES_NAME,
|
486
|
-
help="Select from available compute packages",
|
487
|
-
)
|
488
|
-
|
489
|
-
sp.add_argument(
|
490
|
-
"-c",
|
491
|
-
"--compute",
|
492
|
-
nargs="+",
|
493
|
-
default=None,
|
494
|
-
help="Extra key=value variable pairs",
|
495
|
-
)
|
496
|
-
|
497
|
-
# sp.add_argument(
|
498
|
-
# "-t", "--template",
|
499
|
-
# help="Provide a template file (not yet implemented).")
|
500
|
-
|
501
|
-
sp.add_argument(
|
502
|
-
"-o", "--outfile", required=False, default=None, help="Output filepath"
|
503
|
-
)
|
504
|
-
|
505
|
-
return parser
|
506
|
-
|
507
|
-
|
508
|
-
def main():
|
509
|
-
"""Primary workflow"""
|
510
|
-
|
511
|
-
parser = logmuse.add_logging_options(build_argparser())
|
512
|
-
# args, remaining_args = parser.parse_known_args()
|
513
|
-
args = parser.parse_args()
|
514
|
-
|
515
|
-
logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
|
516
|
-
logmuse.init_logger("yacman", **logger_kwargs)
|
517
|
-
global _LOGGER
|
518
|
-
_LOGGER = logmuse.logger_via_cli(args)
|
519
|
-
|
520
|
-
if not args.command:
|
521
|
-
parser.print_help()
|
522
|
-
_LOGGER.error("No command given")
|
523
|
-
sys.exit(1)
|
524
|
-
|
525
|
-
if args.command == "init":
|
526
|
-
divcfg = args.config
|
527
|
-
_LOGGER.debug("Initializing divvy configuration")
|
528
|
-
is_writable(os.path.dirname(divcfg), check_exist=False)
|
529
|
-
divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH)
|
530
|
-
sys.exit(0)
|
531
|
-
|
532
|
-
_LOGGER.debug("Divvy config: {}".format(args.config))
|
533
|
-
divcfg = select_divvy_config(args.config)
|
534
|
-
_LOGGER.info("Using divvy config: {}".format(divcfg))
|
535
|
-
dcc = ComputingConfiguration(filepath=divcfg)
|
536
|
-
|
537
|
-
if args.command == "list":
|
538
|
-
# Output header via logger and content via print so the user can
|
539
|
-
# redirect the list from stdout if desired without the header as clutter
|
540
|
-
_LOGGER.info("Available compute packages:\n")
|
541
|
-
print("{}".format("\n".join(dcc.list_compute_packages())))
|
542
|
-
sys.exit(1)
|
543
|
-
|
544
|
-
if args.command == "inspect":
|
545
|
-
# Output contents of selected compute package
|
546
|
-
_LOGGER.info("Your compute package template for: " + args.package + "\n")
|
547
|
-
found = False
|
548
|
-
for pkg_name, pkg in dcc.compute_packages.items():
|
549
|
-
if pkg_name == args.package:
|
550
|
-
found = True
|
551
|
-
with open(pkg.submission_template, "r") as f:
|
552
|
-
print(f.read())
|
553
|
-
_LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
|
554
|
-
if pkg_name == "docker":
|
555
|
-
print("Docker args are: " + pkg.docker_args)
|
556
|
-
|
557
|
-
if not found:
|
558
|
-
_LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
|
559
|
-
sys.exit(1)
|
560
|
-
|
561
|
-
# Any non-divvy arguments will be passed along as key-value pairs
|
562
|
-
# that can be used to populate the template.
|
563
|
-
# keys = [str.replace(x, "--", "") for x in remaining_args[::2]]
|
564
|
-
# cli_vars = dict(zip(keys, remaining_args[1::2]))
|
565
|
-
if args.compute:
|
566
|
-
cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]}
|
567
|
-
else:
|
568
|
-
cli_vars = {}
|
569
|
-
|
570
|
-
if args.command == "write" or args.command == "submit":
|
571
|
-
try:
|
572
|
-
dcc.activate_package(args.package)
|
573
|
-
except AttributeError:
|
574
|
-
parser.print_help(sys.stderr)
|
575
|
-
sys.exit(1)
|
576
|
-
|
577
|
-
if args.settings:
|
578
|
-
_LOGGER.info("Loading settings file: %s", args.settings)
|
579
|
-
with open(args.settings, "r") as f:
|
580
|
-
vars_groups = [cli_vars, yaml.load(f, SafeLoader)]
|
581
|
-
else:
|
582
|
-
vars_groups = [cli_vars]
|
583
|
-
|
584
|
-
_LOGGER.debug(vars_groups)
|
585
|
-
if args.command == "write":
|
586
|
-
dcc.write_script(args.outfile, vars_groups)
|
587
|
-
elif args.command == "submit":
|
588
|
-
dcc.submit(args.outfile, vars_groups)
|
looper/exceptions.py
CHANGED
@@ -37,6 +37,13 @@ class MisconfigurationException(LooperError):
|
|
37
37
|
super(MisconfigurationException, self).__init__(key)
|
38
38
|
|
39
39
|
|
40
|
+
class RegistryPathException(LooperError):
|
41
|
+
"""Duplication of pipeline identifier precludes unique pipeline ref."""
|
42
|
+
|
43
|
+
def __init__(self, msg):
|
44
|
+
super(RegistryPathException, self).__init__(msg)
|
45
|
+
|
46
|
+
|
40
47
|
class DuplicatePipelineKeyException(LooperError):
|
41
48
|
"""Duplication of pipeline identifier precludes unique pipeline ref."""
|
42
49
|
|
@@ -60,6 +67,17 @@ class JobSubmissionException(LooperError):
|
|
60
67
|
super(JobSubmissionException, self).__init__(reason)
|
61
68
|
|
62
69
|
|
70
|
+
class PipestatConfigurationException(LooperError):
|
71
|
+
"""Error type for when command fails due to missing pipestat config"""
|
72
|
+
|
73
|
+
def __init__(
|
74
|
+
self,
|
75
|
+
sub_cmd,
|
76
|
+
):
|
77
|
+
reason = "Pipestat must be configured for command {}".format(sub_cmd)
|
78
|
+
super(PipestatConfigurationException, self).__init__(reason)
|
79
|
+
|
80
|
+
|
63
81
|
class MissingPipelineConfigurationException(LooperError):
|
64
82
|
"""A selected pipeline needs configuration data."""
|
65
83
|
|