looper 1.7.0a1__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
looper/looper.py CHANGED
@@ -6,6 +6,7 @@ Looper: a pipeline submission engine. https://github.com/pepkit/looper
6
6
  import abc
7
7
  import argparse
8
8
  import csv
9
+ import glob
9
10
  import logging
10
11
  import subprocess
11
12
  import yaml
@@ -15,6 +16,8 @@ import pandas as _pd
15
16
  # Need specific sequence of actions for colorama imports?
16
17
  from colorama import init
17
18
 
19
+ from .const import PipelineLevel
20
+
18
21
  init()
19
22
  from shutil import rmtree
20
23
 
@@ -30,14 +33,12 @@ from rich.color import Color
30
33
  from rich.console import Console
31
34
  from rich.table import Table
32
35
  from ubiquerg.cli_tools import query_yes_no
33
- from ubiquerg.collection import uniqify
34
36
 
35
37
 
36
38
  from .conductor import SubmissionConductor
37
39
 
38
40
  from .exceptions import *
39
41
  from .const import *
40
- from .pipeline_interface import PipelineInterface
41
42
  from .project import Project
42
43
  from .utils import (
43
44
  desired_samples_range_skipped,
@@ -45,7 +46,7 @@ from .utils import (
45
46
  sample_folder,
46
47
  )
47
48
  from pipestat.reports import get_file_for_table
48
- from pipestat.reports import get_file_for_project
49
+ from pipestat.exceptions import PipestatSummarizeError
49
50
 
50
51
  _PKGNAME = "looper"
51
52
  _LOGGER = logging.getLogger(_PKGNAME)
@@ -88,21 +89,37 @@ class Checker(Executor):
88
89
 
89
90
  # aggregate pipeline status data
90
91
  status = {}
91
- if args.project:
92
- psms = self.prj.get_pipestat_managers(project_level=True)
93
- for pipeline_name, psm in psms.items():
94
- s = psm.get_status() or "unknown"
95
- status.setdefault(pipeline_name, {})
96
- status[pipeline_name][self.prj.name] = s
97
- _LOGGER.debug(f"{self.prj.name} ({pipeline_name}): {s}")
92
+
93
+ psms = {}
94
+ if getattr(args, "project", None):
95
+
96
+ for piface in self.prj.project_pipeline_interfaces:
97
+ if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
98
+ if piface.psm.pipeline_name not in psms:
99
+ psms[piface.psm.pipeline_name] = piface.psm
100
+ for pl_name, psm in psms.items():
101
+ all_project_level_records = psm.select_records()
102
+ for record in all_project_level_records["records"]:
103
+ s = piface.psm.get_status(
104
+ record_identifier=record["record_identifier"]
105
+ )
106
+ status.setdefault(piface.psm.pipeline_name, {})
107
+ status[piface.psm.pipeline_name][record["record_identifier"]] = s
108
+ _LOGGER.debug(
109
+ f"{self.prj.name} ({record['record_identifier']}): {s}"
110
+ )
111
+
98
112
  else:
99
113
  for sample in self.prj.samples:
100
- psms = self.prj.get_pipestat_managers(sample_name=sample.sample_name)
101
- for pipeline_name, psm in psms.items():
102
- s = psm.get_status(record_identifier=sample.sample_name)
103
- status.setdefault(pipeline_name, {})
104
- status[pipeline_name][sample.sample_name] = s
105
- _LOGGER.debug(f"{sample.sample_name} ({pipeline_name}): {s}")
114
+ for piface in sample.project.pipeline_interfaces:
115
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
116
+ psms[piface.psm.pipeline_name] = piface.psm
117
+ s = piface.psm.get_status(record_identifier=sample.sample_name)
118
+ status.setdefault(piface.psm.pipeline_name, {})
119
+ status[piface.psm.pipeline_name][sample.sample_name] = s
120
+ _LOGGER.debug(
121
+ f"{sample.sample_name} ({piface.psm.pipeline_name}): {s}"
122
+ )
106
123
 
107
124
  console = Console()
108
125
 
@@ -116,14 +133,14 @@ class Checker(Executor):
116
133
  )
117
134
  table.add_column(f"Status", justify="center")
118
135
  table.add_column("Jobs count/total jobs", justify="center")
119
- for status_id in psm.status_schema.keys():
136
+ for status_id in psms[pipeline_name].status_schema.keys():
120
137
  status_list = list(pipeline_status.values())
121
138
  if status_id in status_list:
122
139
  status_count = status_list.count(status_id)
123
140
  table.add_row(status_id, f"{status_count}/{len(status_list)}")
124
141
  console.print(table)
125
142
 
126
- if args.itemized:
143
+ if getattr(args, "itemized", None):
127
144
  for pipeline_name, pipeline_status in status.items():
128
145
  table_title = f"Pipeline: '{pipeline_name}'"
129
146
  table = Table(
@@ -141,7 +158,7 @@ class Checker(Executor):
141
158
  for name, status_id in pipeline_status.items():
142
159
  try:
143
160
  color = Color.from_rgb(
144
- *psm.status_schema[status_id]["color"]
161
+ *psms[pipeline_name].status_schema[status_id]["color"]
145
162
  ).name
146
163
  except KeyError:
147
164
  color = "#bcbcbc"
@@ -150,16 +167,17 @@ class Checker(Executor):
150
167
  console.print(table)
151
168
 
152
169
  if args.describe_codes:
170
+ # TODO this needs to be redone because it only takes the last psm in the list and gets status code and descriptions
153
171
  table = Table(
154
172
  show_header=True,
155
173
  header_style="bold magenta",
156
174
  title=f"Status codes description",
157
- width=len(psm.status_schema_source) + 20,
158
- caption=f"Descriptions source: {psm.status_schema_source}",
175
+ width=len(psms[pipeline_name].status_schema_source) + 20,
176
+ caption=f"Descriptions source: {psms[pipeline_name].status_schema_source}",
159
177
  )
160
178
  table.add_column("Status code", justify="center")
161
179
  table.add_column("Description", justify="left")
162
- for status, status_obj in psm.status_schema.items():
180
+ for status, status_obj in psms[pipeline_name].status_schema.items():
163
181
  if "description" in status_obj:
164
182
  desc = status_obj["description"]
165
183
  else:
@@ -199,10 +217,10 @@ class Cleaner(Executor):
199
217
  if not preview_flag:
200
218
  _LOGGER.info("Clean complete.")
201
219
  return 0
202
- if args.dry_run:
220
+ if getattr(args, "dry_run", None):
203
221
  _LOGGER.info("Dry run. No files cleaned.")
204
222
  return 0
205
- if not args.force_yes and not query_yes_no(
223
+ if not getattr(args, "force_yes", None) and not query_yes_no(
206
224
  "Are you sure you want to permanently delete all "
207
225
  "intermediate pipeline results for this project?"
208
226
  ):
@@ -241,8 +259,22 @@ class Destroyer(Executor):
241
259
  :param bool preview_flag: whether to halt before actually removing files
242
260
  """
243
261
 
244
- _LOGGER.info("Removing results:")
262
+ use_pipestat = (
263
+ self.prj.pipestat_configured_project
264
+ if getattr(args, "project", None)
265
+ else self.prj.pipestat_configured
266
+ )
245
267
 
268
+ if use_pipestat:
269
+ _LOGGER.info("Removing summary:")
270
+ destroy_summary(
271
+ self.prj,
272
+ dry_run=preview_flag,
273
+ project_level=getattr(args, "project", None),
274
+ )
275
+
276
+ _LOGGER.info("Removing results:")
277
+ psms = {}
246
278
  for sample in select_samples(prj=self.prj, args=args):
247
279
  _LOGGER.info(self.counter.show(sample.sample_name))
248
280
  sample_output_folder = sample_folder(self.prj, sample)
@@ -250,30 +282,26 @@ class Destroyer(Executor):
250
282
  # Preview: Don't actually delete, just show files.
251
283
  _LOGGER.info(str(sample_output_folder))
252
284
  else:
253
- _remove_or_dry_run(sample_output_folder, args.dry_run)
254
-
255
- _LOGGER.info("Removing summary:")
256
- use_pipestat = (
257
- self.prj.pipestat_configured_project
258
- if args.project
259
- else self.prj.pipestat_configured
260
- )
261
- if use_pipestat:
262
- destroy_summary(self.prj, args.dry_run, args.project)
263
- else:
264
- _LOGGER.warning(
265
- "Pipestat must be configured to destroy any created summaries."
266
- )
285
+ if use_pipestat:
286
+ for piface in sample.project.pipeline_interfaces:
287
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
288
+ psms[piface.psm.pipeline_name] = piface.psm
289
+ for pipeline_name, psm in psms.items():
290
+ psm.backend.remove_record(
291
+ record_identifier=sample.sample_name, rm_record=True
292
+ )
293
+ else:
294
+ _remove_or_dry_run(sample_output_folder, args.dry_run)
267
295
 
268
296
  if not preview_flag:
269
297
  _LOGGER.info("Destroy complete.")
270
298
  return 0
271
299
 
272
- if args.dry_run:
300
+ if getattr(args, "dry_run", None):
273
301
  _LOGGER.info("Dry run. No files destroyed.")
274
302
  return 0
275
303
 
276
- if not args.force_yes and not query_yes_no(
304
+ if not getattr(args, "force_yes", None) and not query_yes_no(
277
305
  "Are you sure you want to permanently delete all pipeline "
278
306
  "results for this project?"
279
307
  ):
@@ -308,7 +336,7 @@ class Collator(Executor):
308
336
  """
309
337
  jobs = 0
310
338
  self.debug = {}
311
- project_pifaces = self.prj.project_pipeline_interface_sources
339
+ project_pifaces = self.prj.project_pipeline_interfaces
312
340
  if not project_pifaces:
313
341
  raise MisconfigurationException(
314
342
  "Looper requires a pointer to at least one project pipeline. "
@@ -318,36 +346,26 @@ class Collator(Executor):
318
346
  )
319
347
  self.counter = LooperCounter(len(project_pifaces))
320
348
  for project_piface in project_pifaces:
321
- try:
322
- project_piface_object = PipelineInterface(
323
- project_piface, pipeline_type="project"
324
- )
325
- except (IOError, ValidationError) as e:
326
- _LOGGER.warning(
327
- "Ignoring invalid pipeline interface source: {}. "
328
- "Caught exception: {}".format(
329
- project_piface, getattr(e, "message", repr(e))
330
- )
331
- )
332
- continue
333
349
  _LOGGER.info(
334
350
  self.counter.show(
335
351
  name=self.prj.name,
336
352
  type="project",
337
- pipeline_name=project_piface_object.pipeline_name,
353
+ pipeline_name=project_piface.pipeline_name,
338
354
  )
339
355
  )
340
356
  conductor = SubmissionConductor(
341
- pipeline_interface=project_piface_object,
357
+ pipeline_interface=project_piface,
342
358
  prj=self.prj,
343
359
  compute_variables=compute_kwargs,
344
- delay=args.time_delay,
345
- extra_args=args.command_extra,
346
- extra_args_override=args.command_extra_override,
347
- ignore_flags=args.ignore_flags,
360
+ delay=getattr(args, "time_delay", None),
361
+ extra_args=getattr(args, "command_extra", None),
362
+ extra_args_override=getattr(args, "command_extra_override", None),
363
+ ignore_flags=getattr(args, "ignore_flags", None),
348
364
  collate=True,
349
365
  )
350
- if conductor.is_project_submittable(force=args.ignore_flags):
366
+ if conductor.is_project_submittable(
367
+ force=getattr(args, "ignore_flags", None)
368
+ ):
351
369
  conductor._pool = [None]
352
370
  conductor.submit()
353
371
  jobs += conductor.num_job_submissions
@@ -360,7 +378,7 @@ class Collator(Executor):
360
378
  class Runner(Executor):
361
379
  """The true submitter of pipelines"""
362
380
 
363
- def __call__(self, args, rerun=False, **compute_kwargs):
381
+ def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
364
382
  """
365
383
  Do the Sample submission.
366
384
 
@@ -395,24 +413,24 @@ class Runner(Executor):
395
413
  )
396
414
 
397
415
  submission_conductors = {}
416
+
398
417
  for piface in self.prj.pipeline_interfaces:
399
418
  conductor = SubmissionConductor(
400
419
  pipeline_interface=piface,
401
420
  prj=self.prj,
402
421
  compute_variables=comp_vars,
403
- delay=args.time_delay,
404
- extra_args=args.command_extra,
405
- extra_args_override=args.command_extra_override,
406
- ignore_flags=args.ignore_flags,
407
- max_cmds=args.lumpn,
408
- max_size=args.lump,
422
+ delay=getattr(args, "time_delay", None),
423
+ extra_args=getattr(args, "command_extra", None),
424
+ extra_args_override=getattr(args, "command_extra_override", None),
425
+ ignore_flags=getattr(args, "ignore_flags", None),
426
+ max_cmds=getattr(args, "lump_n", None),
427
+ max_size=getattr(args, "lump", None),
428
+ max_jobs=getattr(args, "lump_j", None),
409
429
  )
410
430
  submission_conductors[piface.pipe_iface_file] = conductor
411
431
 
412
- _LOGGER.info(f"Pipestat compatible: {self.prj.pipestat_configured_project}")
413
- self.debug["Pipestat compatible"] = (
414
- self.prj.pipestat_configured_project or self.prj.pipestat_configured
415
- )
432
+ _LOGGER.debug(f"Pipestat compatible: {self.prj.pipestat_configured}")
433
+ self.debug["Pipestat compatible"] = self.prj.pipestat_configured
416
434
 
417
435
  for sample in select_samples(prj=self.prj, args=args):
418
436
  pl_fails = []
@@ -484,15 +502,15 @@ class Runner(Executor):
484
502
  len(processed_samples), num_samples
485
503
  )
486
504
  )
487
- _LOGGER.info("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
505
+ _LOGGER.debug("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
488
506
  self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
489
- if args.dry_run:
507
+ if getattr(args, "dry_run", None):
490
508
  job_sub_total_if_real = job_sub_total
491
509
  job_sub_total = 0
492
510
  _LOGGER.info(
493
511
  f"Dry run. No jobs were actually submitted, but {job_sub_total_if_real} would have been."
494
512
  )
495
- _LOGGER.info("Jobs submitted: {}".format(job_sub_total))
513
+ _LOGGER.debug("Jobs submitted: {}".format(job_sub_total))
496
514
  self.debug[DEBUG_JOBS] = job_sub_total
497
515
 
498
516
  # Restructure sample/failure data for display.
@@ -544,37 +562,55 @@ class Reporter(Executor):
544
562
 
545
563
  def __call__(self, args):
546
564
  # initialize the report builder
565
+ self.debug = {}
547
566
  p = self.prj
548
- project_level = args.project
567
+ project_level = getattr(args, "project", None)
549
568
 
550
569
  portable = args.portable
551
570
 
571
+ report_dir = getattr(args, "report_dir", None)
572
+
573
+ psms = {}
574
+
552
575
  if project_level:
553
- psms = self.prj.get_pipestat_managers(project_level=True)
554
- print(psms)
555
- for name, psm in psms.items():
556
- # Summarize will generate the static HTML Report Function
557
- report_directory = psm.summarize(
558
- looper_samples=self.prj.samples, portable=portable
559
- )
576
+
577
+ for piface in self.prj.project_pipeline_interfaces:
578
+ if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
579
+ if piface.psm.pipeline_name not in psms:
580
+ psms[piface.psm.pipeline_name] = piface.psm
581
+ for pl_name, psm in psms.items():
582
+ try:
583
+ report_directory = psm.summarize(
584
+ looper_samples=self.prj.samples,
585
+ portable=portable,
586
+ output_dir=report_dir,
587
+ )
588
+ except PipestatSummarizeError as e:
589
+ raise LooperReportError(
590
+ f"Looper report error due to the following exception: {e}"
591
+ )
560
592
  print(f"Report directory: {report_directory}")
593
+ self.debug["report_directory"] = report_directory
594
+ return self.debug
561
595
  else:
562
- for piface_source_samples in self.prj._samples_by_piface(
563
- self.prj.piface_key
564
- ).values():
565
- # For each piface_key, we have a list of samples, but we only need one sample from the list to
566
- # call the related pipestat manager object which will pull ALL samples when using psm.summarize
567
- first_sample_name = list(piface_source_samples)[0]
568
- psms = self.prj.get_pipestat_managers(
569
- sample_name=first_sample_name, project_level=False
570
- )
571
- print(psms)
572
- for name, psm in psms.items():
573
- # Summarize will generate the static HTML Report Function
596
+ for piface in self.prj.pipeline_interfaces:
597
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
598
+ if piface.psm.pipeline_name not in psms:
599
+ psms[piface.psm.pipeline_name] = piface.psm
600
+ for pl_name, psm in psms.items():
601
+ try:
574
602
  report_directory = psm.summarize(
575
- looper_samples=self.prj.samples, portable=portable
603
+ looper_samples=self.prj.samples,
604
+ portable=portable,
605
+ output_dir=report_dir,
576
606
  )
577
- print(f"Report directory: {report_directory}")
607
+ except PipestatSummarizeError as e:
608
+ raise LooperReportError(
609
+ f"Looper report error due to the following exception: {e}"
610
+ )
611
+ print(f"Report directory: {report_directory}")
612
+ self.debug["report_directory"] = report_directory
613
+ return self.debug
578
614
 
579
615
 
580
616
  class Linker(Executor):
@@ -583,26 +619,22 @@ class Linker(Executor):
583
619
  def __call__(self, args):
584
620
  # initialize the report builder
585
621
  p = self.prj
586
- project_level = args.project
587
- link_dir = args.output_dir
622
+ project_level = getattr(args, "project", None)
623
+ link_dir = getattr(args, "output_dir", None)
624
+
625
+ psms = {}
588
626
 
589
627
  if project_level:
590
- psms = self.prj.get_pipestat_managers(project_level=True)
591
- for name, psm in psms.items():
592
- linked_results_path = psm.link(link_dir=link_dir)
593
- print(f"Linked directory: {linked_results_path}")
628
+ for piface in self.prj.project_pipeline_interfaces:
629
+ if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
630
+ psms[piface.psm.pipeline_name] = piface.psm
631
+ linked_results_path = piface.psm.link(link_dir=link_dir)
632
+ print(f"Linked directory: {linked_results_path}")
594
633
  else:
595
- for piface_source_samples in self.prj._samples_by_piface(
596
- self.prj.piface_key
597
- ).values():
598
- # For each piface_key, we have a list of samples, but we only need one sample from the list to
599
- # call the related pipestat manager object which will pull ALL samples when using psm.summarize
600
- first_sample_name = list(piface_source_samples)[0]
601
- psms = self.prj.get_pipestat_managers(
602
- sample_name=first_sample_name, project_level=False
603
- )
604
- for name, psm in psms.items():
605
- linked_results_path = psm.link(link_dir=link_dir)
634
+ for piface in self.prj.pipeline_interfaces:
635
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
636
+ psms[piface.psm.pipeline_name] = piface.psm
637
+ linked_results_path = piface.psm.link(link_dir=link_dir)
606
638
  print(f"Linked directory: {linked_results_path}")
607
639
 
608
640
 
@@ -614,24 +646,24 @@ class Tabulator(Executor):
614
646
 
615
647
  def __call__(self, args):
616
648
  # p = self.prj
617
- project_level = args.project
649
+ project_level = getattr(args, "project", None)
650
+ report_dir = getattr(args, "report_dir", None)
618
651
  results = []
652
+ psms = {}
619
653
  if project_level:
620
- psms = self.prj.get_pipestat_managers(project_level=True)
621
- for name, psm in psms.items():
622
- results = psm.table()
654
+ for piface in self.prj.project_pipeline_interfaces:
655
+ if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
656
+ if piface.psm.pipeline_name not in psms:
657
+ psms[piface.psm.pipeline_name] = piface.psm
658
+ for pl_name, psm in psms.items():
659
+ results = psm.table(output_dir=report_dir)
623
660
  else:
624
- for piface_source_samples in self.prj._samples_by_piface(
625
- self.prj.piface_key
626
- ).values():
627
- # For each piface_key, we have a list of samples, but we only need one sample from the list to
628
- # call the related pipestat manager object which will pull ALL samples when using psm.table
629
- first_sample_name = list(piface_source_samples)[0]
630
- psms = self.prj.get_pipestat_managers(
631
- sample_name=first_sample_name, project_level=False
632
- )
633
- for name, psm in psms.items():
634
- results = psm.table()
661
+ for piface in self.prj.pipeline_interfaces:
662
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
663
+ if piface.psm.pipeline_name not in psms:
664
+ psms[piface.psm.pipeline_name] = piface.psm
665
+ for pl_name, psm in psms.items():
666
+ results = psm.table(output_dir=report_dir)
635
667
  # Results contains paths to stats and object summaries.
636
668
  return results
637
669
 
@@ -671,64 +703,60 @@ def destroy_summary(prj, dry_run=False, project_level=False):
671
703
  This function is for use with pipestat configured projects.
672
704
  """
673
705
 
706
+ psms = {}
674
707
  if project_level:
675
- psms = prj.get_pipestat_managers(project_level=True)
708
+ for piface in prj.pipeline_interfaces:
709
+ if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
710
+ psms[piface.psm.pipeline_name] = piface.psm
711
+
676
712
  for name, psm in psms.items():
677
713
  _remove_or_dry_run(
678
714
  [
679
- get_file_for_project(
680
- psm,
681
- pipeline_name=psm["_pipeline_name"],
682
- directory="reports",
715
+ get_file_for_table(
716
+ psm, pipeline_name=psm.pipeline_name, directory="reports"
683
717
  ),
684
718
  get_file_for_table(
685
719
  psm,
686
- pipeline_name=psm["_pipeline_name"],
720
+ pipeline_name=psm.pipeline_name,
687
721
  appendix="stats_summary.tsv",
688
722
  ),
689
723
  get_file_for_table(
690
724
  psm,
691
- pipeline_name=psm["_pipeline_name"],
725
+ pipeline_name=psm.pipeline_name,
692
726
  appendix="objs_summary.yaml",
693
727
  ),
694
- get_file_for_table(
695
- psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
728
+ os.path.join(
729
+ os.path.dirname(psm.config_path), "aggregate_results.yaml"
696
730
  ),
697
731
  ],
698
732
  dry_run,
699
733
  )
700
734
  else:
701
- for piface_source_samples in prj._samples_by_piface(prj.piface_key).values():
702
- # For each piface_key, we have a list of samples, but we only need one sample from the list to
703
- # call the related pipestat manager object which will pull ALL samples when using psm.table
704
- first_sample_name = list(piface_source_samples)[0]
705
- psms = prj.get_pipestat_managers(
706
- sample_name=first_sample_name, project_level=False
735
+ for piface in prj.pipeline_interfaces:
736
+ if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
737
+ psms[piface.psm.pipeline_name] = piface.psm
738
+ for name, psm in psms.items():
739
+ _remove_or_dry_run(
740
+ [
741
+ get_file_for_table(
742
+ psm, pipeline_name=psm.pipeline_name, directory="reports"
743
+ ),
744
+ get_file_for_table(
745
+ psm,
746
+ pipeline_name=psm.pipeline_name,
747
+ appendix="stats_summary.tsv",
748
+ ),
749
+ get_file_for_table(
750
+ psm,
751
+ pipeline_name=psm.pipeline_name,
752
+ appendix="objs_summary.yaml",
753
+ ),
754
+ os.path.join(
755
+ os.path.dirname(psm.config_path), "aggregate_results.yaml"
756
+ ),
757
+ ],
758
+ dry_run,
707
759
  )
708
- for name, psm in psms.items():
709
- _remove_or_dry_run(
710
- [
711
- get_file_for_project(
712
- psm,
713
- pipeline_name=psm["_pipeline_name"],
714
- directory="reports",
715
- ),
716
- get_file_for_table(
717
- psm,
718
- pipeline_name=psm["_pipeline_name"],
719
- appendix="stats_summary.tsv",
720
- ),
721
- get_file_for_table(
722
- psm,
723
- pipeline_name=psm["_pipeline_name"],
724
- appendix="objs_summary.yaml",
725
- ),
726
- get_file_for_table(
727
- psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
728
- ),
729
- ],
730
- dry_run,
731
- )
732
760
 
733
761
 
734
762
  class LooperCounter(object):
@@ -17,7 +17,7 @@ from .exceptions import (
17
17
  InvalidResourceSpecificationException,
18
18
  PipelineInterfaceConfigError,
19
19
  )
20
- from .utils import jinja_render_template_strictly
20
+ from .utils import render_nested_var_templates
21
21
 
22
22
  __author__ = "Michal Stolarczyk"
23
23
  __email__ = "michal@virginia.edu"
@@ -56,15 +56,6 @@ class PipelineInterface(YAMLConfigManager):
56
56
  )
57
57
  self.update(config)
58
58
  self._validate(schema_src=PIFACE_SCHEMA_SRC)
59
- if "path" in self:
60
- warn(
61
- message="'path' specification as a top-level pipeline "
62
- "interface key is deprecated and will be removed with "
63
- "the next release. Please use 'paths' section "
64
- "from now on.",
65
- category=DeprecationWarning,
66
- )
67
- self._expand_paths(["path"])
68
59
  self._expand_paths(["compute", "dynamic_variables_script_path"])
69
60
 
70
61
  @property
@@ -89,8 +80,7 @@ class PipelineInterface(YAMLConfigManager):
89
80
  var_templates = {}
90
81
  if curr_data:
91
82
  var_templates.update(curr_data)
92
- for k, v in var_templates.items():
93
- var_templates[k] = jinja_render_template_strictly(v, namespaces)
83
+ var_templates = render_nested_var_templates(var_templates, namespaces)
94
84
  return var_templates
95
85
 
96
86
  def get_pipeline_schemas(self, schema_key=INPUT_SCHEMA_KEY):