looper 1.7.0a1__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/__main__.py +1 -1
- looper/_version.py +2 -1
- looper/cli_divvy.py +10 -6
- looper/cli_pydantic.py +413 -0
- looper/command_models/DEVELOPER.md +85 -0
- looper/command_models/README.md +4 -0
- looper/command_models/__init__.py +6 -0
- looper/command_models/arguments.py +293 -0
- looper/command_models/commands.py +335 -0
- looper/conductor.py +161 -28
- looper/const.py +9 -0
- looper/divvy.py +56 -47
- looper/exceptions.py +9 -1
- looper/looper.py +196 -168
- looper/pipeline_interface.py +2 -12
- looper/project.py +154 -176
- looper/schemas/pipeline_interface_schema_generic.yaml +14 -6
- looper/utils.py +450 -78
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/METADATA +24 -14
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/RECORD +24 -19
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/WHEEL +1 -1
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/entry_points.txt +1 -1
- looper/cli_looper.py +0 -788
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/LICENSE.txt +0 -0
- {looper-1.7.0a1.dist-info → looper-2.0.0.dist-info}/top_level.txt +0 -0
looper/looper.py
CHANGED
@@ -6,6 +6,7 @@ Looper: a pipeline submission engine. https://github.com/pepkit/looper
|
|
6
6
|
import abc
|
7
7
|
import argparse
|
8
8
|
import csv
|
9
|
+
import glob
|
9
10
|
import logging
|
10
11
|
import subprocess
|
11
12
|
import yaml
|
@@ -15,6 +16,8 @@ import pandas as _pd
|
|
15
16
|
# Need specific sequence of actions for colorama imports?
|
16
17
|
from colorama import init
|
17
18
|
|
19
|
+
from .const import PipelineLevel
|
20
|
+
|
18
21
|
init()
|
19
22
|
from shutil import rmtree
|
20
23
|
|
@@ -30,14 +33,12 @@ from rich.color import Color
|
|
30
33
|
from rich.console import Console
|
31
34
|
from rich.table import Table
|
32
35
|
from ubiquerg.cli_tools import query_yes_no
|
33
|
-
from ubiquerg.collection import uniqify
|
34
36
|
|
35
37
|
|
36
38
|
from .conductor import SubmissionConductor
|
37
39
|
|
38
40
|
from .exceptions import *
|
39
41
|
from .const import *
|
40
|
-
from .pipeline_interface import PipelineInterface
|
41
42
|
from .project import Project
|
42
43
|
from .utils import (
|
43
44
|
desired_samples_range_skipped,
|
@@ -45,7 +46,7 @@ from .utils import (
|
|
45
46
|
sample_folder,
|
46
47
|
)
|
47
48
|
from pipestat.reports import get_file_for_table
|
48
|
-
from pipestat.
|
49
|
+
from pipestat.exceptions import PipestatSummarizeError
|
49
50
|
|
50
51
|
_PKGNAME = "looper"
|
51
52
|
_LOGGER = logging.getLogger(_PKGNAME)
|
@@ -88,21 +89,37 @@ class Checker(Executor):
|
|
88
89
|
|
89
90
|
# aggregate pipeline status data
|
90
91
|
status = {}
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
92
|
+
|
93
|
+
psms = {}
|
94
|
+
if getattr(args, "project", None):
|
95
|
+
|
96
|
+
for piface in self.prj.project_pipeline_interfaces:
|
97
|
+
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
98
|
+
if piface.psm.pipeline_name not in psms:
|
99
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
100
|
+
for pl_name, psm in psms.items():
|
101
|
+
all_project_level_records = psm.select_records()
|
102
|
+
for record in all_project_level_records["records"]:
|
103
|
+
s = piface.psm.get_status(
|
104
|
+
record_identifier=record["record_identifier"]
|
105
|
+
)
|
106
|
+
status.setdefault(piface.psm.pipeline_name, {})
|
107
|
+
status[piface.psm.pipeline_name][record["record_identifier"]] = s
|
108
|
+
_LOGGER.debug(
|
109
|
+
f"{self.prj.name} ({record['record_identifier']}): {s}"
|
110
|
+
)
|
111
|
+
|
98
112
|
else:
|
99
113
|
for sample in self.prj.samples:
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
114
|
+
for piface in sample.project.pipeline_interfaces:
|
115
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
116
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
117
|
+
s = piface.psm.get_status(record_identifier=sample.sample_name)
|
118
|
+
status.setdefault(piface.psm.pipeline_name, {})
|
119
|
+
status[piface.psm.pipeline_name][sample.sample_name] = s
|
120
|
+
_LOGGER.debug(
|
121
|
+
f"{sample.sample_name} ({piface.psm.pipeline_name}): {s}"
|
122
|
+
)
|
106
123
|
|
107
124
|
console = Console()
|
108
125
|
|
@@ -116,14 +133,14 @@ class Checker(Executor):
|
|
116
133
|
)
|
117
134
|
table.add_column(f"Status", justify="center")
|
118
135
|
table.add_column("Jobs count/total jobs", justify="center")
|
119
|
-
for status_id in
|
136
|
+
for status_id in psms[pipeline_name].status_schema.keys():
|
120
137
|
status_list = list(pipeline_status.values())
|
121
138
|
if status_id in status_list:
|
122
139
|
status_count = status_list.count(status_id)
|
123
140
|
table.add_row(status_id, f"{status_count}/{len(status_list)}")
|
124
141
|
console.print(table)
|
125
142
|
|
126
|
-
if args
|
143
|
+
if getattr(args, "itemized", None):
|
127
144
|
for pipeline_name, pipeline_status in status.items():
|
128
145
|
table_title = f"Pipeline: '{pipeline_name}'"
|
129
146
|
table = Table(
|
@@ -141,7 +158,7 @@ class Checker(Executor):
|
|
141
158
|
for name, status_id in pipeline_status.items():
|
142
159
|
try:
|
143
160
|
color = Color.from_rgb(
|
144
|
-
*
|
161
|
+
*psms[pipeline_name].status_schema[status_id]["color"]
|
145
162
|
).name
|
146
163
|
except KeyError:
|
147
164
|
color = "#bcbcbc"
|
@@ -150,16 +167,17 @@ class Checker(Executor):
|
|
150
167
|
console.print(table)
|
151
168
|
|
152
169
|
if args.describe_codes:
|
170
|
+
# TODO this needs to be redone because it only takes the last psm in the list and gets status code and descriptions
|
153
171
|
table = Table(
|
154
172
|
show_header=True,
|
155
173
|
header_style="bold magenta",
|
156
174
|
title=f"Status codes description",
|
157
|
-
width=len(
|
158
|
-
caption=f"Descriptions source: {
|
175
|
+
width=len(psms[pipeline_name].status_schema_source) + 20,
|
176
|
+
caption=f"Descriptions source: {psms[pipeline_name].status_schema_source}",
|
159
177
|
)
|
160
178
|
table.add_column("Status code", justify="center")
|
161
179
|
table.add_column("Description", justify="left")
|
162
|
-
for status, status_obj in
|
180
|
+
for status, status_obj in psms[pipeline_name].status_schema.items():
|
163
181
|
if "description" in status_obj:
|
164
182
|
desc = status_obj["description"]
|
165
183
|
else:
|
@@ -199,10 +217,10 @@ class Cleaner(Executor):
|
|
199
217
|
if not preview_flag:
|
200
218
|
_LOGGER.info("Clean complete.")
|
201
219
|
return 0
|
202
|
-
if args
|
220
|
+
if getattr(args, "dry_run", None):
|
203
221
|
_LOGGER.info("Dry run. No files cleaned.")
|
204
222
|
return 0
|
205
|
-
if not args
|
223
|
+
if not getattr(args, "force_yes", None) and not query_yes_no(
|
206
224
|
"Are you sure you want to permanently delete all "
|
207
225
|
"intermediate pipeline results for this project?"
|
208
226
|
):
|
@@ -241,8 +259,22 @@ class Destroyer(Executor):
|
|
241
259
|
:param bool preview_flag: whether to halt before actually removing files
|
242
260
|
"""
|
243
261
|
|
244
|
-
|
262
|
+
use_pipestat = (
|
263
|
+
self.prj.pipestat_configured_project
|
264
|
+
if getattr(args, "project", None)
|
265
|
+
else self.prj.pipestat_configured
|
266
|
+
)
|
245
267
|
|
268
|
+
if use_pipestat:
|
269
|
+
_LOGGER.info("Removing summary:")
|
270
|
+
destroy_summary(
|
271
|
+
self.prj,
|
272
|
+
dry_run=preview_flag,
|
273
|
+
project_level=getattr(args, "project", None),
|
274
|
+
)
|
275
|
+
|
276
|
+
_LOGGER.info("Removing results:")
|
277
|
+
psms = {}
|
246
278
|
for sample in select_samples(prj=self.prj, args=args):
|
247
279
|
_LOGGER.info(self.counter.show(sample.sample_name))
|
248
280
|
sample_output_folder = sample_folder(self.prj, sample)
|
@@ -250,30 +282,26 @@ class Destroyer(Executor):
|
|
250
282
|
# Preview: Don't actually delete, just show files.
|
251
283
|
_LOGGER.info(str(sample_output_folder))
|
252
284
|
else:
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
else:
|
264
|
-
_LOGGER.warning(
|
265
|
-
"Pipestat must be configured to destroy any created summaries."
|
266
|
-
)
|
285
|
+
if use_pipestat:
|
286
|
+
for piface in sample.project.pipeline_interfaces:
|
287
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
288
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
289
|
+
for pipeline_name, psm in psms.items():
|
290
|
+
psm.backend.remove_record(
|
291
|
+
record_identifier=sample.sample_name, rm_record=True
|
292
|
+
)
|
293
|
+
else:
|
294
|
+
_remove_or_dry_run(sample_output_folder, args.dry_run)
|
267
295
|
|
268
296
|
if not preview_flag:
|
269
297
|
_LOGGER.info("Destroy complete.")
|
270
298
|
return 0
|
271
299
|
|
272
|
-
if args
|
300
|
+
if getattr(args, "dry_run", None):
|
273
301
|
_LOGGER.info("Dry run. No files destroyed.")
|
274
302
|
return 0
|
275
303
|
|
276
|
-
if not args
|
304
|
+
if not getattr(args, "force_yes", None) and not query_yes_no(
|
277
305
|
"Are you sure you want to permanently delete all pipeline "
|
278
306
|
"results for this project?"
|
279
307
|
):
|
@@ -308,7 +336,7 @@ class Collator(Executor):
|
|
308
336
|
"""
|
309
337
|
jobs = 0
|
310
338
|
self.debug = {}
|
311
|
-
project_pifaces = self.prj.
|
339
|
+
project_pifaces = self.prj.project_pipeline_interfaces
|
312
340
|
if not project_pifaces:
|
313
341
|
raise MisconfigurationException(
|
314
342
|
"Looper requires a pointer to at least one project pipeline. "
|
@@ -318,36 +346,26 @@ class Collator(Executor):
|
|
318
346
|
)
|
319
347
|
self.counter = LooperCounter(len(project_pifaces))
|
320
348
|
for project_piface in project_pifaces:
|
321
|
-
try:
|
322
|
-
project_piface_object = PipelineInterface(
|
323
|
-
project_piface, pipeline_type="project"
|
324
|
-
)
|
325
|
-
except (IOError, ValidationError) as e:
|
326
|
-
_LOGGER.warning(
|
327
|
-
"Ignoring invalid pipeline interface source: {}. "
|
328
|
-
"Caught exception: {}".format(
|
329
|
-
project_piface, getattr(e, "message", repr(e))
|
330
|
-
)
|
331
|
-
)
|
332
|
-
continue
|
333
349
|
_LOGGER.info(
|
334
350
|
self.counter.show(
|
335
351
|
name=self.prj.name,
|
336
352
|
type="project",
|
337
|
-
pipeline_name=
|
353
|
+
pipeline_name=project_piface.pipeline_name,
|
338
354
|
)
|
339
355
|
)
|
340
356
|
conductor = SubmissionConductor(
|
341
|
-
pipeline_interface=
|
357
|
+
pipeline_interface=project_piface,
|
342
358
|
prj=self.prj,
|
343
359
|
compute_variables=compute_kwargs,
|
344
|
-
delay=args
|
345
|
-
extra_args=args
|
346
|
-
extra_args_override=args
|
347
|
-
ignore_flags=args
|
360
|
+
delay=getattr(args, "time_delay", None),
|
361
|
+
extra_args=getattr(args, "command_extra", None),
|
362
|
+
extra_args_override=getattr(args, "command_extra_override", None),
|
363
|
+
ignore_flags=getattr(args, "ignore_flags", None),
|
348
364
|
collate=True,
|
349
365
|
)
|
350
|
-
if conductor.is_project_submittable(
|
366
|
+
if conductor.is_project_submittable(
|
367
|
+
force=getattr(args, "ignore_flags", None)
|
368
|
+
):
|
351
369
|
conductor._pool = [None]
|
352
370
|
conductor.submit()
|
353
371
|
jobs += conductor.num_job_submissions
|
@@ -360,7 +378,7 @@ class Collator(Executor):
|
|
360
378
|
class Runner(Executor):
|
361
379
|
"""The true submitter of pipelines"""
|
362
380
|
|
363
|
-
def __call__(self, args, rerun=False, **compute_kwargs):
|
381
|
+
def __call__(self, args, top_level_args=None, rerun=False, **compute_kwargs):
|
364
382
|
"""
|
365
383
|
Do the Sample submission.
|
366
384
|
|
@@ -395,24 +413,24 @@ class Runner(Executor):
|
|
395
413
|
)
|
396
414
|
|
397
415
|
submission_conductors = {}
|
416
|
+
|
398
417
|
for piface in self.prj.pipeline_interfaces:
|
399
418
|
conductor = SubmissionConductor(
|
400
419
|
pipeline_interface=piface,
|
401
420
|
prj=self.prj,
|
402
421
|
compute_variables=comp_vars,
|
403
|
-
delay=args
|
404
|
-
extra_args=args
|
405
|
-
extra_args_override=args
|
406
|
-
ignore_flags=args
|
407
|
-
max_cmds=args
|
408
|
-
max_size=args
|
422
|
+
delay=getattr(args, "time_delay", None),
|
423
|
+
extra_args=getattr(args, "command_extra", None),
|
424
|
+
extra_args_override=getattr(args, "command_extra_override", None),
|
425
|
+
ignore_flags=getattr(args, "ignore_flags", None),
|
426
|
+
max_cmds=getattr(args, "lump_n", None),
|
427
|
+
max_size=getattr(args, "lump", None),
|
428
|
+
max_jobs=getattr(args, "lump_j", None),
|
409
429
|
)
|
410
430
|
submission_conductors[piface.pipe_iface_file] = conductor
|
411
431
|
|
412
|
-
_LOGGER.
|
413
|
-
self.debug["Pipestat compatible"] =
|
414
|
-
self.prj.pipestat_configured_project or self.prj.pipestat_configured
|
415
|
-
)
|
432
|
+
_LOGGER.debug(f"Pipestat compatible: {self.prj.pipestat_configured}")
|
433
|
+
self.debug["Pipestat compatible"] = self.prj.pipestat_configured
|
416
434
|
|
417
435
|
for sample in select_samples(prj=self.prj, args=args):
|
418
436
|
pl_fails = []
|
@@ -484,15 +502,15 @@ class Runner(Executor):
|
|
484
502
|
len(processed_samples), num_samples
|
485
503
|
)
|
486
504
|
)
|
487
|
-
_LOGGER.
|
505
|
+
_LOGGER.debug("Commands submitted: {} of {}".format(cmd_sub_total, max_cmds))
|
488
506
|
self.debug[DEBUG_COMMANDS] = "{} of {}".format(cmd_sub_total, max_cmds)
|
489
|
-
if args
|
507
|
+
if getattr(args, "dry_run", None):
|
490
508
|
job_sub_total_if_real = job_sub_total
|
491
509
|
job_sub_total = 0
|
492
510
|
_LOGGER.info(
|
493
511
|
f"Dry run. No jobs were actually submitted, but {job_sub_total_if_real} would have been."
|
494
512
|
)
|
495
|
-
_LOGGER.
|
513
|
+
_LOGGER.debug("Jobs submitted: {}".format(job_sub_total))
|
496
514
|
self.debug[DEBUG_JOBS] = job_sub_total
|
497
515
|
|
498
516
|
# Restructure sample/failure data for display.
|
@@ -544,37 +562,55 @@ class Reporter(Executor):
|
|
544
562
|
|
545
563
|
def __call__(self, args):
|
546
564
|
# initialize the report builder
|
565
|
+
self.debug = {}
|
547
566
|
p = self.prj
|
548
|
-
project_level = args
|
567
|
+
project_level = getattr(args, "project", None)
|
549
568
|
|
550
569
|
portable = args.portable
|
551
570
|
|
571
|
+
report_dir = getattr(args, "report_dir", None)
|
572
|
+
|
573
|
+
psms = {}
|
574
|
+
|
552
575
|
if project_level:
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
576
|
+
|
577
|
+
for piface in self.prj.project_pipeline_interfaces:
|
578
|
+
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
579
|
+
if piface.psm.pipeline_name not in psms:
|
580
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
581
|
+
for pl_name, psm in psms.items():
|
582
|
+
try:
|
583
|
+
report_directory = psm.summarize(
|
584
|
+
looper_samples=self.prj.samples,
|
585
|
+
portable=portable,
|
586
|
+
output_dir=report_dir,
|
587
|
+
)
|
588
|
+
except PipestatSummarizeError as e:
|
589
|
+
raise LooperReportError(
|
590
|
+
f"Looper report error due to the following exception: {e}"
|
591
|
+
)
|
560
592
|
print(f"Report directory: {report_directory}")
|
593
|
+
self.debug["report_directory"] = report_directory
|
594
|
+
return self.debug
|
561
595
|
else:
|
562
|
-
for
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
psms = self.prj.get_pipestat_managers(
|
569
|
-
sample_name=first_sample_name, project_level=False
|
570
|
-
)
|
571
|
-
print(psms)
|
572
|
-
for name, psm in psms.items():
|
573
|
-
# Summarize will generate the static HTML Report Function
|
596
|
+
for piface in self.prj.pipeline_interfaces:
|
597
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
598
|
+
if piface.psm.pipeline_name not in psms:
|
599
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
600
|
+
for pl_name, psm in psms.items():
|
601
|
+
try:
|
574
602
|
report_directory = psm.summarize(
|
575
|
-
looper_samples=self.prj.samples,
|
603
|
+
looper_samples=self.prj.samples,
|
604
|
+
portable=portable,
|
605
|
+
output_dir=report_dir,
|
576
606
|
)
|
577
|
-
|
607
|
+
except PipestatSummarizeError as e:
|
608
|
+
raise LooperReportError(
|
609
|
+
f"Looper report error due to the following exception: {e}"
|
610
|
+
)
|
611
|
+
print(f"Report directory: {report_directory}")
|
612
|
+
self.debug["report_directory"] = report_directory
|
613
|
+
return self.debug
|
578
614
|
|
579
615
|
|
580
616
|
class Linker(Executor):
|
@@ -583,26 +619,22 @@ class Linker(Executor):
|
|
583
619
|
def __call__(self, args):
|
584
620
|
# initialize the report builder
|
585
621
|
p = self.prj
|
586
|
-
project_level = args
|
587
|
-
link_dir = args
|
622
|
+
project_level = getattr(args, "project", None)
|
623
|
+
link_dir = getattr(args, "output_dir", None)
|
624
|
+
|
625
|
+
psms = {}
|
588
626
|
|
589
627
|
if project_level:
|
590
|
-
|
591
|
-
|
592
|
-
|
593
|
-
|
628
|
+
for piface in self.prj.project_pipeline_interfaces:
|
629
|
+
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
630
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
631
|
+
linked_results_path = piface.psm.link(link_dir=link_dir)
|
632
|
+
print(f"Linked directory: {linked_results_path}")
|
594
633
|
else:
|
595
|
-
for
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
# call the related pipestat manager object which will pull ALL samples when using psm.summarize
|
600
|
-
first_sample_name = list(piface_source_samples)[0]
|
601
|
-
psms = self.prj.get_pipestat_managers(
|
602
|
-
sample_name=first_sample_name, project_level=False
|
603
|
-
)
|
604
|
-
for name, psm in psms.items():
|
605
|
-
linked_results_path = psm.link(link_dir=link_dir)
|
634
|
+
for piface in self.prj.pipeline_interfaces:
|
635
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
636
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
637
|
+
linked_results_path = piface.psm.link(link_dir=link_dir)
|
606
638
|
print(f"Linked directory: {linked_results_path}")
|
607
639
|
|
608
640
|
|
@@ -614,24 +646,24 @@ class Tabulator(Executor):
|
|
614
646
|
|
615
647
|
def __call__(self, args):
|
616
648
|
# p = self.prj
|
617
|
-
project_level = args
|
649
|
+
project_level = getattr(args, "project", None)
|
650
|
+
report_dir = getattr(args, "report_dir", None)
|
618
651
|
results = []
|
652
|
+
psms = {}
|
619
653
|
if project_level:
|
620
|
-
|
621
|
-
|
622
|
-
|
654
|
+
for piface in self.prj.project_pipeline_interfaces:
|
655
|
+
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
656
|
+
if piface.psm.pipeline_name not in psms:
|
657
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
658
|
+
for pl_name, psm in psms.items():
|
659
|
+
results = psm.table(output_dir=report_dir)
|
623
660
|
else:
|
624
|
-
for
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
psms = self.prj.get_pipestat_managers(
|
631
|
-
sample_name=first_sample_name, project_level=False
|
632
|
-
)
|
633
|
-
for name, psm in psms.items():
|
634
|
-
results = psm.table()
|
661
|
+
for piface in self.prj.pipeline_interfaces:
|
662
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
663
|
+
if piface.psm.pipeline_name not in psms:
|
664
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
665
|
+
for pl_name, psm in psms.items():
|
666
|
+
results = psm.table(output_dir=report_dir)
|
635
667
|
# Results contains paths to stats and object summaries.
|
636
668
|
return results
|
637
669
|
|
@@ -671,64 +703,60 @@ def destroy_summary(prj, dry_run=False, project_level=False):
|
|
671
703
|
This function is for use with pipestat configured projects.
|
672
704
|
"""
|
673
705
|
|
706
|
+
psms = {}
|
674
707
|
if project_level:
|
675
|
-
|
708
|
+
for piface in prj.pipeline_interfaces:
|
709
|
+
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
710
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
711
|
+
|
676
712
|
for name, psm in psms.items():
|
677
713
|
_remove_or_dry_run(
|
678
714
|
[
|
679
|
-
|
680
|
-
psm,
|
681
|
-
pipeline_name=psm["_pipeline_name"],
|
682
|
-
directory="reports",
|
715
|
+
get_file_for_table(
|
716
|
+
psm, pipeline_name=psm.pipeline_name, directory="reports"
|
683
717
|
),
|
684
718
|
get_file_for_table(
|
685
719
|
psm,
|
686
|
-
pipeline_name=psm
|
720
|
+
pipeline_name=psm.pipeline_name,
|
687
721
|
appendix="stats_summary.tsv",
|
688
722
|
),
|
689
723
|
get_file_for_table(
|
690
724
|
psm,
|
691
|
-
pipeline_name=psm
|
725
|
+
pipeline_name=psm.pipeline_name,
|
692
726
|
appendix="objs_summary.yaml",
|
693
727
|
),
|
694
|
-
|
695
|
-
psm,
|
728
|
+
os.path.join(
|
729
|
+
os.path.dirname(psm.config_path), "aggregate_results.yaml"
|
696
730
|
),
|
697
731
|
],
|
698
732
|
dry_run,
|
699
733
|
)
|
700
734
|
else:
|
701
|
-
for
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
735
|
+
for piface in prj.pipeline_interfaces:
|
736
|
+
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
737
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
738
|
+
for name, psm in psms.items():
|
739
|
+
_remove_or_dry_run(
|
740
|
+
[
|
741
|
+
get_file_for_table(
|
742
|
+
psm, pipeline_name=psm.pipeline_name, directory="reports"
|
743
|
+
),
|
744
|
+
get_file_for_table(
|
745
|
+
psm,
|
746
|
+
pipeline_name=psm.pipeline_name,
|
747
|
+
appendix="stats_summary.tsv",
|
748
|
+
),
|
749
|
+
get_file_for_table(
|
750
|
+
psm,
|
751
|
+
pipeline_name=psm.pipeline_name,
|
752
|
+
appendix="objs_summary.yaml",
|
753
|
+
),
|
754
|
+
os.path.join(
|
755
|
+
os.path.dirname(psm.config_path), "aggregate_results.yaml"
|
756
|
+
),
|
757
|
+
],
|
758
|
+
dry_run,
|
707
759
|
)
|
708
|
-
for name, psm in psms.items():
|
709
|
-
_remove_or_dry_run(
|
710
|
-
[
|
711
|
-
get_file_for_project(
|
712
|
-
psm,
|
713
|
-
pipeline_name=psm["_pipeline_name"],
|
714
|
-
directory="reports",
|
715
|
-
),
|
716
|
-
get_file_for_table(
|
717
|
-
psm,
|
718
|
-
pipeline_name=psm["_pipeline_name"],
|
719
|
-
appendix="stats_summary.tsv",
|
720
|
-
),
|
721
|
-
get_file_for_table(
|
722
|
-
psm,
|
723
|
-
pipeline_name=psm["_pipeline_name"],
|
724
|
-
appendix="objs_summary.yaml",
|
725
|
-
),
|
726
|
-
get_file_for_table(
|
727
|
-
psm, pipeline_name=psm["_pipeline_name"], appendix="reports"
|
728
|
-
),
|
729
|
-
],
|
730
|
-
dry_run,
|
731
|
-
)
|
732
760
|
|
733
761
|
|
734
762
|
class LooperCounter(object):
|
looper/pipeline_interface.py
CHANGED
@@ -17,7 +17,7 @@ from .exceptions import (
|
|
17
17
|
InvalidResourceSpecificationException,
|
18
18
|
PipelineInterfaceConfigError,
|
19
19
|
)
|
20
|
-
from .utils import
|
20
|
+
from .utils import render_nested_var_templates
|
21
21
|
|
22
22
|
__author__ = "Michal Stolarczyk"
|
23
23
|
__email__ = "michal@virginia.edu"
|
@@ -56,15 +56,6 @@ class PipelineInterface(YAMLConfigManager):
|
|
56
56
|
)
|
57
57
|
self.update(config)
|
58
58
|
self._validate(schema_src=PIFACE_SCHEMA_SRC)
|
59
|
-
if "path" in self:
|
60
|
-
warn(
|
61
|
-
message="'path' specification as a top-level pipeline "
|
62
|
-
"interface key is deprecated and will be removed with "
|
63
|
-
"the next release. Please use 'paths' section "
|
64
|
-
"from now on.",
|
65
|
-
category=DeprecationWarning,
|
66
|
-
)
|
67
|
-
self._expand_paths(["path"])
|
68
59
|
self._expand_paths(["compute", "dynamic_variables_script_path"])
|
69
60
|
|
70
61
|
@property
|
@@ -89,8 +80,7 @@ class PipelineInterface(YAMLConfigManager):
|
|
89
80
|
var_templates = {}
|
90
81
|
if curr_data:
|
91
82
|
var_templates.update(curr_data)
|
92
|
-
|
93
|
-
var_templates[k] = jinja_render_template_strictly(v, namespaces)
|
83
|
+
var_templates = render_nested_var_templates(var_templates, namespaces)
|
94
84
|
return var_templates
|
95
85
|
|
96
86
|
def get_pipeline_schemas(self, schema_key=INPUT_SCHEMA_KEY):
|