looper 1.5.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
looper/divvy.py CHANGED
@@ -28,7 +28,7 @@ _LOGGER = logging.getLogger(__name__)
28
28
  # This is the divvy.py submodule from divvy
29
29
 
30
30
 
31
- class ComputingConfiguration(yacman.YacAttMap):
31
+ class ComputingConfiguration(yacman.YAMLConfigManager):
32
32
  """
33
33
  Represents computing configuration objects.
34
34
 
@@ -53,30 +53,30 @@ class ComputingConfiguration(yacman.YacAttMap):
53
53
  entries=entries,
54
54
  filepath=filepath,
55
55
  schema_source=DEFAULT_CONFIG_SCHEMA,
56
- write_validate=True,
56
+ validate_on_write=True,
57
57
  )
58
58
 
59
- if not hasattr(self, "compute_packages"):
59
+ if not "compute_packages" in self:
60
60
  raise Exception(
61
61
  "Your divvy config file is not in divvy config format "
62
62
  "(it lacks a compute_packages section): '{}'".format(filepath)
63
63
  )
64
64
  # We require that compute_packages be present, even if empty
65
- self.compute_packages = {}
65
+ self["compute_packages"] = {}
66
66
 
67
67
  # Initialize default compute settings.
68
68
  _LOGGER.debug("Establishing project compute settings")
69
69
  self.compute = None
70
70
  self.setdefault("adapters", None)
71
71
  self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
72
- self.config_file = self["__internal"].file_path
72
+ self.config_file = self.filepath
73
73
 
74
74
  def write(self, filename=None):
75
75
  super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
76
76
  filename = filename or getattr(self, yacman.FILEPATH_KEY)
77
77
  filedir = os.path.dirname(filename)
78
78
  # For this object, we *also* have to write the template files
79
- for pkg_name, pkg in self.compute_packages.items():
79
+ for pkg_name, pkg in self["compute_packages"].items():
80
80
  print(pkg)
81
81
  destfile = os.path.join(filedir, os.path.basename(pkg.submission_template))
82
82
  shutil.copyfile(pkg.submission_template, destfile)
@@ -109,7 +109,7 @@ class ComputingConfiguration(yacman.YacAttMap):
109
109
 
110
110
  :return str: submission script content template for current state
111
111
  """
112
- with open(self.compute.submission_template, "r") as f:
112
+ with open(self.compute["submission_template"], "r") as f:
113
113
  return f.read()
114
114
 
115
115
  @property
@@ -145,28 +145,28 @@ class ComputingConfiguration(yacman.YacAttMap):
145
145
 
146
146
  if (
147
147
  package_name
148
- and self.compute_packages
149
- and package_name in self.compute_packages
148
+ and self["compute_packages"]
149
+ and package_name in self["compute_packages"]
150
150
  ):
151
151
  # Augment compute, creating it if needed.
152
152
  if self.compute is None:
153
153
  _LOGGER.debug("Creating Project compute")
154
- self.compute = yacman.YacAttMap()
154
+ self.compute = yacman.YAMLConfigManager()
155
155
  _LOGGER.debug(
156
156
  "Adding entries for package_name '{}'".format(package_name)
157
157
  )
158
158
 
159
- self.compute.add_entries(self.compute_packages[package_name])
159
+ self.compute.update(self["compute_packages"][package_name])
160
160
 
161
161
  # Ensure submission template is absolute. This *used to be* handled
162
162
  # at update (so the paths were stored as absolutes in the packages),
163
163
  # but now, it makes more sense to do it here so we can piggyback on
164
164
  # the default update() method and not even have to do that.
165
- if not os.path.isabs(self.compute.submission_template):
165
+ if not os.path.isabs(self.compute["submission_template"]):
166
166
  try:
167
- self.compute.submission_template = os.path.join(
168
- os.path.dirname(self["__internal"].file_path),
169
- self.compute.submission_template,
167
+ self.compute["submission_template"] = os.path.join(
168
+ os.path.dirname(self.filepath),
169
+ self.compute["submission_template"],
170
170
  )
171
171
  except AttributeError as e:
172
172
  # Environment and environment compute should at least have been
@@ -174,7 +174,7 @@ class ComputingConfiguration(yacman.YacAttMap):
174
174
  _LOGGER.error(str(e))
175
175
 
176
176
  _LOGGER.debug(
177
- "Submit template set to: {}".format(self.compute.submission_template)
177
+ "Submit template set to: {}".format(self.compute["submission_template"])
178
178
  )
179
179
 
180
180
  return True
@@ -184,7 +184,7 @@ class ComputingConfiguration(yacman.YacAttMap):
184
184
  # both present--but don't evaluate to True--is fairly harmless.
185
185
  _LOGGER.debug(
186
186
  "Can't activate package. compute_packages = {}".format(
187
- self.compute_packages
187
+ self["compute_packages"]
188
188
  )
189
189
  )
190
190
 
@@ -214,7 +214,7 @@ class ComputingConfiguration(yacman.YacAttMap):
214
214
 
215
215
  :return set[str]: names of available compute packages
216
216
  """
217
- return set(self.compute_packages.keys())
217
+ return set(self["compute_packages"].keys())
218
218
 
219
219
  def reset_active_settings(self):
220
220
  """
@@ -248,13 +248,13 @@ class ComputingConfiguration(yacman.YacAttMap):
248
248
  package-specific set of adapters, if any defined in 'adapters' section
249
249
  under currently active compute package.
250
250
 
251
- :return yacman.YacAttMap: current adapters mapping
251
+ :return yacman.YAMLConfigManager: current adapters mapping
252
252
  """
253
- adapters = yacman.YacAttMap()
254
- if "adapters" in self and self.adapters is not None:
255
- adapters.update(self.adapters)
253
+ adapters = yacman.YAMLConfigManager()
254
+ if "adapters" in self and self["adapters"] is not None:
255
+ adapters.update(self["adapters"])
256
256
  if "compute" in self and "adapters" in self.compute:
257
- adapters.update(self.compute.adapters)
257
+ adapters.update(self.compute["adapters"])
258
258
  if not adapters:
259
259
  _LOGGER.debug("No adapters determined in divvy configuration file.")
260
260
  return adapters
@@ -270,7 +270,9 @@ class ComputingConfiguration(yacman.YacAttMap):
270
270
  self.submit(temp.name, extra_vars)
271
271
  else:
272
272
  script = self.write_script(output_path, extra_vars)
273
- submission_command = "{} {}".format(self.compute.submission_command, script)
273
+ submission_command = "{} {}".format(
274
+ self.compute["submission_command"], script
275
+ )
274
276
  _LOGGER.info(submission_command)
275
277
  os.system(submission_command)
276
278
 
@@ -337,7 +339,7 @@ class ComputingConfiguration(yacman.YacAttMap):
337
339
  if len(extra_var) > 0 and list(extra_var.keys())[0] not in exclude:
338
340
  variables.update(extra_var)
339
341
  _LOGGER.debug(
340
- "Submission template: {}".format(self.compute.submission_template)
342
+ "Submission template: {}".format(self.compute["submission_template"])
341
343
  )
342
344
  if output_path:
343
345
  _LOGGER.info("Writing script to {}".format(os.path.abspath(output_path)))
@@ -379,6 +381,7 @@ def select_divvy_config(filepath):
379
381
  config_env_vars=COMPUTE_SETTINGS_VARNAME,
380
382
  default_config_filepath=DEFAULT_CONFIG_FILEPATH,
381
383
  check_exist=True,
384
+ config_name="divvy",
382
385
  )
383
386
  _LOGGER.debug("Selected divvy config: {}".format(divcfg))
384
387
  return divcfg
@@ -415,174 +418,3 @@ def divvy_init(config_path, template_config_path):
415
418
  _LOGGER.info("Wrote new divvy configuration file: {}".format(config_path))
416
419
  else:
417
420
  _LOGGER.warning("Can't initialize, file exists: {} ".format(config_path))
418
-
419
-
420
- def build_argparser():
421
- """
422
- Builds argument parser.
423
-
424
- :return argparse.ArgumentParser
425
- """
426
-
427
- banner = (
428
- "%(prog)s - write compute job scripts that can be submitted to "
429
- "any computing resource"
430
- )
431
- additional_description = "\nhttps://divvy.databio.org"
432
-
433
- parser = VersionInHelpParser(
434
- prog="divvy",
435
- description=banner,
436
- epilog=additional_description,
437
- # version=__version__,
438
- )
439
-
440
- subparsers = parser.add_subparsers(dest="command")
441
-
442
- def add_subparser(cmd, description):
443
- return subparsers.add_parser(cmd, description=description, help=description)
444
-
445
- subparser_messages = {
446
- "init": "Initialize a new divvy config file",
447
- "list": "List available compute packages",
448
- "write": "Write a job script",
449
- "submit": "Write and then submit a job script",
450
- "inspect": "Inspect compute package",
451
- }
452
-
453
- sps = {}
454
- for cmd, desc in subparser_messages.items():
455
- sps[cmd] = add_subparser(cmd, desc)
456
- # sps[cmd].add_argument(
457
- # "config", nargs="?", default=None,
458
- # help="Divvy configuration file.")
459
-
460
- for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
461
- sp.add_argument(
462
- "config", nargs="?", default=None, help="Divvy configuration file."
463
- )
464
-
465
- sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
466
-
467
- for sp in [sps["inspect"]]:
468
- sp.add_argument(
469
- "-p",
470
- "--package",
471
- default=DEFAULT_COMPUTE_RESOURCES_NAME,
472
- help="Select from available compute packages",
473
- )
474
-
475
- for sp in [sps["write"], sps["submit"]]:
476
- sp.add_argument(
477
- "-s",
478
- "--settings",
479
- help="YAML file with job settings to populate the template",
480
- )
481
-
482
- sp.add_argument(
483
- "-p",
484
- "--package",
485
- default=DEFAULT_COMPUTE_RESOURCES_NAME,
486
- help="Select from available compute packages",
487
- )
488
-
489
- sp.add_argument(
490
- "-c",
491
- "--compute",
492
- nargs="+",
493
- default=None,
494
- help="Extra key=value variable pairs",
495
- )
496
-
497
- # sp.add_argument(
498
- # "-t", "--template",
499
- # help="Provide a template file (not yet implemented).")
500
-
501
- sp.add_argument(
502
- "-o", "--outfile", required=False, default=None, help="Output filepath"
503
- )
504
-
505
- return parser
506
-
507
-
508
- def main():
509
- """Primary workflow"""
510
-
511
- parser = logmuse.add_logging_options(build_argparser())
512
- # args, remaining_args = parser.parse_known_args()
513
- args = parser.parse_args()
514
-
515
- logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
516
- logmuse.init_logger("yacman", **logger_kwargs)
517
- global _LOGGER
518
- _LOGGER = logmuse.logger_via_cli(args)
519
-
520
- if not args.command:
521
- parser.print_help()
522
- _LOGGER.error("No command given")
523
- sys.exit(1)
524
-
525
- if args.command == "init":
526
- divcfg = args.config
527
- _LOGGER.debug("Initializing divvy configuration")
528
- is_writable(os.path.dirname(divcfg), check_exist=False)
529
- divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH)
530
- sys.exit(0)
531
-
532
- _LOGGER.debug("Divvy config: {}".format(args.config))
533
- divcfg = select_divvy_config(args.config)
534
- _LOGGER.info("Using divvy config: {}".format(divcfg))
535
- dcc = ComputingConfiguration(filepath=divcfg)
536
-
537
- if args.command == "list":
538
- # Output header via logger and content via print so the user can
539
- # redirect the list from stdout if desired without the header as clutter
540
- _LOGGER.info("Available compute packages:\n")
541
- print("{}".format("\n".join(dcc.list_compute_packages())))
542
- sys.exit(1)
543
-
544
- if args.command == "inspect":
545
- # Output contents of selected compute package
546
- _LOGGER.info("Your compute package template for: " + args.package + "\n")
547
- found = False
548
- for pkg_name, pkg in dcc.compute_packages.items():
549
- if pkg_name == args.package:
550
- found = True
551
- with open(pkg.submission_template, "r") as f:
552
- print(f.read())
553
- _LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
554
- if pkg_name == "docker":
555
- print("Docker args are: " + pkg.docker_args)
556
-
557
- if not found:
558
- _LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
559
- sys.exit(1)
560
-
561
- # Any non-divvy arguments will be passed along as key-value pairs
562
- # that can be used to populate the template.
563
- # keys = [str.replace(x, "--", "") for x in remaining_args[::2]]
564
- # cli_vars = dict(zip(keys, remaining_args[1::2]))
565
- if args.compute:
566
- cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]}
567
- else:
568
- cli_vars = {}
569
-
570
- if args.command == "write" or args.command == "submit":
571
- try:
572
- dcc.activate_package(args.package)
573
- except AttributeError:
574
- parser.print_help(sys.stderr)
575
- sys.exit(1)
576
-
577
- if args.settings:
578
- _LOGGER.info("Loading settings file: %s", args.settings)
579
- with open(args.settings, "r") as f:
580
- vars_groups = [cli_vars, yaml.load(f, SafeLoader)]
581
- else:
582
- vars_groups = [cli_vars]
583
-
584
- _LOGGER.debug(vars_groups)
585
- if args.command == "write":
586
- dcc.write_script(args.outfile, vars_groups)
587
- elif args.command == "submit":
588
- dcc.submit(args.outfile, vars_groups)
looper/exceptions.py CHANGED
@@ -37,6 +37,13 @@ class MisconfigurationException(LooperError):
37
37
  super(MisconfigurationException, self).__init__(key)
38
38
 
39
39
 
40
+ class RegistryPathException(LooperError):
41
+ """Duplication of pipeline identifier precludes unique pipeline ref."""
42
+
43
+ def __init__(self, msg):
44
+ super(RegistryPathException, self).__init__(msg)
45
+
46
+
40
47
  class DuplicatePipelineKeyException(LooperError):
41
48
  """Duplication of pipeline identifier precludes unique pipeline ref."""
42
49
 
@@ -60,6 +67,17 @@ class JobSubmissionException(LooperError):
60
67
  super(JobSubmissionException, self).__init__(reason)
61
68
 
62
69
 
70
+ class PipestatConfigurationException(LooperError):
71
+ """Error type for when command fails due to missing pipestat config"""
72
+
73
+ def __init__(
74
+ self,
75
+ sub_cmd,
76
+ ):
77
+ reason = "Pipestat must be configured for command {}".format(sub_cmd)
78
+ super(PipestatConfigurationException, self).__init__(reason)
79
+
80
+
63
81
  class MissingPipelineConfigurationException(LooperError):
64
82
  """A selected pipeline needs configuration data."""
65
83