looper 1.7.0__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/__main__.py +1 -1
- looper/_version.py +2 -1
- looper/cli_divvy.py +10 -6
- looper/cli_pydantic.py +413 -0
- looper/command_models/DEVELOPER.md +85 -0
- looper/command_models/README.md +4 -0
- looper/command_models/__init__.py +6 -0
- looper/command_models/arguments.py +293 -0
- looper/command_models/commands.py +335 -0
- looper/conductor.py +147 -28
- looper/const.py +9 -0
- looper/divvy.py +56 -47
- looper/exceptions.py +9 -1
- looper/looper.py +196 -169
- looper/pipeline_interface.py +2 -12
- looper/project.py +154 -176
- looper/schemas/pipeline_interface_schema_generic.yaml +14 -6
- looper/utils.py +450 -78
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/METADATA +24 -14
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/RECORD +24 -19
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/WHEEL +1 -1
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/entry_points.txt +1 -1
- looper/cli_looper.py +0 -796
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/LICENSE.txt +0 -0
- {looper-1.7.0.dist-info → looper-2.0.0.dist-info}/top_level.txt +0 -0
looper/utils.py
CHANGED
@@ -1,12 +1,11 @@
|
|
1
1
|
""" Helpers without an obvious logical home. """
|
2
2
|
|
3
3
|
import argparse
|
4
|
-
from collections import defaultdict
|
4
|
+
from collections import defaultdict
|
5
5
|
import glob
|
6
6
|
import itertools
|
7
7
|
from logging import getLogger
|
8
8
|
import os
|
9
|
-
import sys
|
10
9
|
from typing import *
|
11
10
|
import re
|
12
11
|
|
@@ -14,12 +13,17 @@ import jinja2
|
|
14
13
|
import yaml
|
15
14
|
from peppy import Project as peppyProject
|
16
15
|
from peppy.const import *
|
17
|
-
from ubiquerg import convert_value, expandpath, parse_registry_path
|
16
|
+
from ubiquerg import convert_value, expandpath, parse_registry_path, deep_update
|
18
17
|
from pephubclient.constants import RegistryPath
|
19
|
-
from pydantic
|
18
|
+
from pydantic import ValidationError
|
19
|
+
from yacman import load_yaml
|
20
|
+
from yaml.parser import ParserError
|
20
21
|
|
21
22
|
from .const import *
|
22
|
-
from .
|
23
|
+
from .command_models.commands import SUPPORTED_COMMANDS
|
24
|
+
from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
|
25
|
+
from rich.console import Console
|
26
|
+
from rich.pretty import pprint
|
23
27
|
|
24
28
|
_LOGGER = getLogger(__name__)
|
25
29
|
|
@@ -94,7 +98,9 @@ def fetch_sample_flags(prj, sample, pl_name, flag_dir=None):
|
|
94
98
|
return [
|
95
99
|
x
|
96
100
|
for x in folder_contents
|
97
|
-
if os.path.splitext(x)[1] == ".flag"
|
101
|
+
if os.path.splitext(x)[1] == ".flag"
|
102
|
+
and os.path.basename(x).startswith(pl_name)
|
103
|
+
and sample.sample_name in x
|
98
104
|
]
|
99
105
|
|
100
106
|
|
@@ -250,22 +256,54 @@ def read_yaml_file(filepath):
|
|
250
256
|
return data
|
251
257
|
|
252
258
|
|
253
|
-
def enrich_args_via_cfg(
|
259
|
+
def enrich_args_via_cfg(
|
260
|
+
subcommand_name,
|
261
|
+
parser_args,
|
262
|
+
aux_parser,
|
263
|
+
test_args=None,
|
264
|
+
cli_modifiers=None,
|
265
|
+
):
|
254
266
|
"""
|
255
|
-
Read in a looper dotfile and set arguments.
|
267
|
+
Read in a looper dotfile, pep config and set arguments.
|
256
268
|
|
257
|
-
Priority order: CLI > dotfile/config > parser default
|
269
|
+
Priority order: CLI > dotfile/config > pep_config > parser default
|
258
270
|
|
271
|
+
:param subcommand name: the name of the command used
|
259
272
|
:param argparse.Namespace parser_args: parsed args by the original parser
|
260
|
-
:param argparse.Namespace aux_parser: parsed args by the
|
273
|
+
:param argparse.Namespace aux_parser: parsed args by the argument parser
|
261
274
|
with defaults suppressed
|
275
|
+
:param dict test_args: dict of args used for pytesting
|
276
|
+
:param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file
|
262
277
|
:return argparse.Namespace: selected argument values
|
263
278
|
"""
|
279
|
+
|
280
|
+
# Did the user provide arguments in the PEP config?
|
264
281
|
cfg_args_all = (
|
265
|
-
_get_subcommand_args(parser_args)
|
266
|
-
if os.path.exists(parser_args.
|
282
|
+
_get_subcommand_args(subcommand_name, parser_args)
|
283
|
+
if os.path.exists(parser_args.pep_config)
|
267
284
|
else dict()
|
268
285
|
)
|
286
|
+
if not cfg_args_all:
|
287
|
+
cfg_args_all = {}
|
288
|
+
|
289
|
+
# Did the user provide arguments/modifiers in the looper config file?
|
290
|
+
looper_config_cli_modifiers = None
|
291
|
+
if cli_modifiers:
|
292
|
+
if str(subcommand_name) in cli_modifiers:
|
293
|
+
looper_config_cli_modifiers = cli_modifiers[subcommand_name]
|
294
|
+
looper_config_cli_modifiers = (
|
295
|
+
{k.replace("-", "_"): v for k, v in looper_config_cli_modifiers.items()}
|
296
|
+
if looper_config_cli_modifiers
|
297
|
+
else None
|
298
|
+
)
|
299
|
+
|
300
|
+
if looper_config_cli_modifiers:
|
301
|
+
_LOGGER.warning(
|
302
|
+
"CLI modifiers were provided in Looper Config and in PEP Project Config. Merging..."
|
303
|
+
)
|
304
|
+
deep_update(cfg_args_all, looper_config_cli_modifiers)
|
305
|
+
_LOGGER.debug(msg=f"Merged CLI modifiers: {cfg_args_all}")
|
306
|
+
|
269
307
|
result = argparse.Namespace()
|
270
308
|
if test_args:
|
271
309
|
cli_args, _ = aux_parser.parse_known_args(args=test_args)
|
@@ -273,23 +311,51 @@ def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
|
|
273
311
|
else:
|
274
312
|
cli_args, _ = aux_parser.parse_known_args()
|
275
313
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
314
|
+
# If any CLI args were provided, make sure they take priority
|
315
|
+
if cli_args:
|
316
|
+
r = getattr(cli_args, subcommand_name)
|
317
|
+
for k, v in cfg_args_all.items():
|
318
|
+
if k in r:
|
319
|
+
cfg_args_all[k] = getattr(r, k)
|
320
|
+
|
321
|
+
def set_single_arg(argname, default_source_namespace, result_namespace):
|
322
|
+
if argname not in POSITIONAL or not hasattr(result, argname):
|
323
|
+
if argname in cli_args:
|
324
|
+
cli_provided_value = getattr(cli_args, argname)
|
325
|
+
r = (
|
326
|
+
convert_value(cli_provided_value)
|
327
|
+
if isinstance(cli_provided_value, str)
|
328
|
+
else cli_provided_value
|
329
|
+
)
|
330
|
+
elif cfg_args_all is not None and argname in cfg_args_all:
|
331
|
+
if isinstance(cfg_args_all[argname], list):
|
332
|
+
r = [convert_value(i) for i in cfg_args_all[argname]]
|
333
|
+
elif isinstance(cfg_args_all[argname], dict):
|
334
|
+
r = cfg_args_all[argname]
|
284
335
|
else:
|
285
|
-
r = convert_value(cfg_args_all[
|
336
|
+
r = convert_value(cfg_args_all[argname])
|
286
337
|
else:
|
287
|
-
r = getattr(
|
288
|
-
setattr(
|
338
|
+
r = getattr(default_source_namespace, argname)
|
339
|
+
setattr(result_namespace, argname, r)
|
340
|
+
|
341
|
+
for top_level_argname in vars(parser_args):
|
342
|
+
if top_level_argname not in [cmd.name for cmd in SUPPORTED_COMMANDS]:
|
343
|
+
# this argument is a top-level argument
|
344
|
+
set_single_arg(top_level_argname, parser_args, result)
|
345
|
+
else:
|
346
|
+
# this argument actually is a subcommand
|
347
|
+
enriched_command_namespace = argparse.Namespace()
|
348
|
+
command_namespace = getattr(parser_args, top_level_argname)
|
349
|
+
if command_namespace:
|
350
|
+
for argname in vars(command_namespace):
|
351
|
+
set_single_arg(
|
352
|
+
argname, command_namespace, enriched_command_namespace
|
353
|
+
)
|
354
|
+
setattr(result, top_level_argname, enriched_command_namespace)
|
289
355
|
return result
|
290
356
|
|
291
357
|
|
292
|
-
def _get_subcommand_args(parser_args):
|
358
|
+
def _get_subcommand_args(subcommand_name, parser_args):
|
293
359
|
"""
|
294
360
|
Get the union of values for the subcommand arguments from
|
295
361
|
Project.looper, Project.looper.cli.<subcommand> and Project.looper.cli.all.
|
@@ -304,7 +370,7 @@ def _get_subcommand_args(parser_args):
|
|
304
370
|
"""
|
305
371
|
args = dict()
|
306
372
|
cfg = peppyProject(
|
307
|
-
parser_args.
|
373
|
+
parser_args.pep_config,
|
308
374
|
defer_samples_creation=True,
|
309
375
|
amendments=parser_args.amend,
|
310
376
|
)
|
@@ -321,8 +387,8 @@ def _get_subcommand_args(parser_args):
|
|
321
387
|
else dict()
|
322
388
|
)
|
323
389
|
args.update(
|
324
|
-
cfg_args[
|
325
|
-
if
|
390
|
+
cfg_args[subcommand_name] or dict()
|
391
|
+
if subcommand_name in cfg_args
|
326
392
|
else dict()
|
327
393
|
)
|
328
394
|
except (TypeError, KeyError, AttributeError, ValueError) as e:
|
@@ -346,40 +412,65 @@ def _get_subcommand_args(parser_args):
|
|
346
412
|
return args
|
347
413
|
|
348
414
|
|
349
|
-
def init_generic_pipeline():
|
415
|
+
def init_generic_pipeline(pipelinepath: Optional[str] = None):
|
350
416
|
"""
|
351
417
|
Create generic pipeline interface
|
352
418
|
"""
|
353
|
-
|
354
|
-
os.makedirs("pipeline")
|
355
|
-
except FileExistsError:
|
356
|
-
pass
|
419
|
+
console = Console()
|
357
420
|
|
358
421
|
# Destination one level down from CWD in pipeline folder
|
359
|
-
|
422
|
+
if not pipelinepath:
|
423
|
+
try:
|
424
|
+
os.makedirs("pipeline")
|
425
|
+
except FileExistsError:
|
426
|
+
pass
|
427
|
+
|
428
|
+
dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
|
429
|
+
else:
|
430
|
+
if os.path.isabs(pipelinepath):
|
431
|
+
dest_file = pipelinepath
|
432
|
+
else:
|
433
|
+
dest_file = os.path.join(os.getcwd(), os.path.relpath(pipelinepath))
|
434
|
+
try:
|
435
|
+
os.makedirs(os.path.dirname(dest_file))
|
436
|
+
except FileExistsError:
|
437
|
+
pass
|
360
438
|
|
361
439
|
# Create Generic Pipeline Interface
|
362
440
|
generic_pipeline_dict = {
|
363
441
|
"pipeline_name": "default_pipeline_name",
|
364
|
-
"pipeline_type": "sample",
|
365
442
|
"output_schema": "output_schema.yaml",
|
366
|
-
"
|
367
|
-
|
368
|
-
|
443
|
+
"sample_interface": {
|
444
|
+
"command_template": "{looper.piface_dir}/count_lines.sh {sample.file} "
|
445
|
+
"--output-parent {looper.sample_output_folder}"
|
446
|
+
},
|
369
447
|
}
|
370
448
|
|
449
|
+
console.rule(f"\n[magenta]Pipeline Interface[/magenta]")
|
371
450
|
# Write file
|
372
451
|
if not os.path.exists(dest_file):
|
452
|
+
pprint(generic_pipeline_dict, expand_all=True)
|
453
|
+
|
373
454
|
with open(dest_file, "w") as file:
|
374
455
|
yaml.dump(generic_pipeline_dict, file)
|
375
|
-
|
456
|
+
|
457
|
+
console.print(
|
458
|
+
f"Pipeline interface successfully created at: [yellow]{dest_file}[/yellow]"
|
459
|
+
)
|
460
|
+
|
376
461
|
else:
|
377
|
-
print(
|
378
|
-
f"Pipeline interface file already exists `{dest_file}
|
462
|
+
console.print(
|
463
|
+
f"Pipeline interface file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
|
379
464
|
)
|
380
465
|
|
381
466
|
# Create Generic Output Schema
|
382
|
-
|
467
|
+
if not pipelinepath:
|
468
|
+
dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
|
469
|
+
else:
|
470
|
+
dest_file = os.path.join(
|
471
|
+
os.path.dirname(dest_file), LOOPER_GENERIC_OUTPUT_SCHEMA
|
472
|
+
)
|
473
|
+
|
383
474
|
generic_output_schema_dict = {
|
384
475
|
"pipeline_name": "default_pipeline_name",
|
385
476
|
"samples": {
|
@@ -389,27 +480,45 @@ def init_generic_pipeline():
|
|
389
480
|
}
|
390
481
|
},
|
391
482
|
}
|
483
|
+
|
484
|
+
console.rule(f"\n[magenta]Output Schema[/magenta]")
|
392
485
|
# Write file
|
393
486
|
if not os.path.exists(dest_file):
|
487
|
+
pprint(generic_output_schema_dict, expand_all=True)
|
394
488
|
with open(dest_file, "w") as file:
|
395
489
|
yaml.dump(generic_output_schema_dict, file)
|
396
|
-
print(
|
490
|
+
console.print(
|
491
|
+
f"Output schema successfully created at: [yellow]{dest_file}[/yellow]"
|
492
|
+
)
|
397
493
|
else:
|
398
|
-
print(
|
494
|
+
console.print(
|
495
|
+
f"Output schema file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
|
496
|
+
)
|
399
497
|
|
498
|
+
console.rule(f"\n[magenta]Example Pipeline Shell Script[/magenta]")
|
400
499
|
# Create Generic countlines.sh
|
401
|
-
|
500
|
+
|
501
|
+
if not pipelinepath:
|
502
|
+
dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
|
503
|
+
else:
|
504
|
+
dest_file = os.path.join(os.path.dirname(dest_file), LOOPER_GENERIC_COUNT_LINES)
|
505
|
+
|
402
506
|
shell_code = """#!/bin/bash
|
403
507
|
linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
|
404
508
|
pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
|
405
509
|
echo "Number of lines: $linecount"
|
406
510
|
"""
|
407
511
|
if not os.path.exists(dest_file):
|
512
|
+
console.print(shell_code)
|
408
513
|
with open(dest_file, "w") as file:
|
409
514
|
file.write(shell_code)
|
410
|
-
print(
|
515
|
+
console.print(
|
516
|
+
f"count_lines.sh successfully created at: [yellow]{dest_file}[/yellow]"
|
517
|
+
)
|
411
518
|
else:
|
412
|
-
print(
|
519
|
+
console.print(
|
520
|
+
f"count_lines.sh file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
|
521
|
+
)
|
413
522
|
|
414
523
|
return True
|
415
524
|
|
@@ -444,12 +553,18 @@ def initiate_looper_config(
|
|
444
553
|
:param bool force: whether the existing file should be overwritten
|
445
554
|
:return bool: whether the file was initialized
|
446
555
|
"""
|
556
|
+
console = Console()
|
557
|
+
console.clear()
|
558
|
+
console.rule(f"\n[magenta]Looper initialization[/magenta]")
|
559
|
+
|
447
560
|
if os.path.exists(looper_config_path) and not force:
|
448
|
-
print(
|
561
|
+
console.print(
|
562
|
+
f"[red]Can't initialize, file exists:[/red] [yellow]{looper_config_path}[/yellow]"
|
563
|
+
)
|
449
564
|
return False
|
450
565
|
|
451
566
|
if pep_path:
|
452
|
-
if
|
567
|
+
if is_pephub_registry_path(pep_path):
|
453
568
|
pass
|
454
569
|
else:
|
455
570
|
pep_path = expandpath(pep_path)
|
@@ -465,21 +580,196 @@ def initiate_looper_config(
|
|
465
580
|
if not output_dir:
|
466
581
|
output_dir = "."
|
467
582
|
|
583
|
+
if sample_pipeline_interfaces is None or sample_pipeline_interfaces == []:
|
584
|
+
sample_pipeline_interfaces = "pipeline_interface1.yaml"
|
585
|
+
|
586
|
+
if project_pipeline_interfaces is None or project_pipeline_interfaces == []:
|
587
|
+
project_pipeline_interfaces = "pipeline_interface2.yaml"
|
588
|
+
|
468
589
|
looper_config_dict = {
|
469
590
|
"pep_config": os.path.relpath(pep_path),
|
470
591
|
"output_dir": output_dir,
|
471
|
-
"pipeline_interfaces":
|
472
|
-
|
473
|
-
|
474
|
-
|
592
|
+
"pipeline_interfaces": [
|
593
|
+
sample_pipeline_interfaces,
|
594
|
+
project_pipeline_interfaces,
|
595
|
+
],
|
475
596
|
}
|
476
597
|
|
598
|
+
pprint(looper_config_dict, expand_all=True)
|
599
|
+
|
477
600
|
with open(looper_config_path, "w") as dotfile:
|
478
601
|
yaml.dump(looper_config_dict, dotfile)
|
479
|
-
print(
|
602
|
+
console.print(
|
603
|
+
f"Initialized looper config file: [yellow]{looper_config_path}[/yellow]"
|
604
|
+
)
|
605
|
+
|
606
|
+
return True
|
607
|
+
|
608
|
+
|
609
|
+
def looper_config_tutorial():
|
610
|
+
"""
|
611
|
+
Prompt a user through configuring a .looper.yaml file for a new project.
|
612
|
+
|
613
|
+
:return bool: whether the file was initialized
|
614
|
+
"""
|
615
|
+
|
616
|
+
console = Console()
|
617
|
+
console.clear()
|
618
|
+
console.rule(f"\n[magenta]Looper initialization[/magenta]")
|
619
|
+
|
620
|
+
looper_cfg_path = ".looper.yaml" # not changeable
|
621
|
+
|
622
|
+
if os.path.exists(looper_cfg_path):
|
623
|
+
console.print(
|
624
|
+
f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]"
|
625
|
+
)
|
626
|
+
raise SystemExit
|
627
|
+
|
628
|
+
cfg = {}
|
629
|
+
|
630
|
+
console.print(
|
631
|
+
"This utility will walk you through creating a [yellow].looper.yaml[/yellow] file."
|
632
|
+
)
|
633
|
+
console.print("See [yellow]`looper init --help`[/yellow] for details.")
|
634
|
+
console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
|
635
|
+
console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
|
636
|
+
|
637
|
+
DEFAULTS = { # What you get if you just press enter
|
638
|
+
"pep_config": "databio/example",
|
639
|
+
"output_dir": "results",
|
640
|
+
"piface_path": "pipeline/pipeline_interface.yaml",
|
641
|
+
"project_name": os.path.basename(os.getcwd()),
|
642
|
+
}
|
643
|
+
|
644
|
+
cfg["project_name"] = (
|
645
|
+
console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >")
|
646
|
+
or DEFAULTS["project_name"]
|
647
|
+
)
|
648
|
+
|
649
|
+
cfg["pep_config"] = (
|
650
|
+
console.input(
|
651
|
+
f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
|
652
|
+
)
|
653
|
+
or DEFAULTS["pep_config"]
|
654
|
+
)
|
655
|
+
|
656
|
+
if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path(
|
657
|
+
cfg["pep_config"]
|
658
|
+
):
|
659
|
+
console.print(
|
660
|
+
f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
|
661
|
+
)
|
662
|
+
|
663
|
+
cfg["output_dir"] = (
|
664
|
+
console.input(
|
665
|
+
f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
|
666
|
+
)
|
667
|
+
or DEFAULTS["output_dir"]
|
668
|
+
)
|
669
|
+
|
670
|
+
add_more_pifaces = True
|
671
|
+
piface_paths = []
|
672
|
+
while add_more_pifaces:
|
673
|
+
piface_path = (
|
674
|
+
console.input(
|
675
|
+
"Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
|
676
|
+
)
|
677
|
+
or None
|
678
|
+
)
|
679
|
+
if piface_path is None:
|
680
|
+
if piface_paths == []:
|
681
|
+
piface_paths.append(DEFAULTS["piface_path"])
|
682
|
+
add_more_pifaces = False
|
683
|
+
else:
|
684
|
+
piface_paths.append(piface_path)
|
685
|
+
|
686
|
+
console.print("\n")
|
687
|
+
|
688
|
+
console.print(
|
689
|
+
f"""\
|
690
|
+
[yellow]pep_config:[/yellow] {cfg['pep_config']}
|
691
|
+
[yellow]output_dir:[/yellow] {cfg['output_dir']}
|
692
|
+
[yellow]pipeline_interfaces:[/yellow]
|
693
|
+
- {piface_paths}
|
694
|
+
"""
|
695
|
+
)
|
696
|
+
|
697
|
+
for piface_path in piface_paths:
|
698
|
+
if not os.path.exists(piface_path):
|
699
|
+
console.print(
|
700
|
+
f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]"
|
701
|
+
)
|
702
|
+
console.print(
|
703
|
+
"Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
|
704
|
+
)
|
705
|
+
selection = None
|
706
|
+
while selection not in ["y", "n"]:
|
707
|
+
selection = console.input("\nSelection: ").lower().strip()
|
708
|
+
if selection == "n":
|
709
|
+
console.print(
|
710
|
+
"Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
|
711
|
+
)
|
712
|
+
if selection == "y":
|
713
|
+
init_generic_pipeline(pipelinepath=piface_path)
|
714
|
+
|
715
|
+
console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
|
716
|
+
|
717
|
+
looper_config_dict = {}
|
718
|
+
looper_config_dict["pep_config"] = cfg["pep_config"]
|
719
|
+
looper_config_dict["output_dir"] = cfg["output_dir"]
|
720
|
+
looper_config_dict["pipeline_interfaces"] = piface_paths
|
721
|
+
|
722
|
+
with open(looper_cfg_path, "w") as fp:
|
723
|
+
yaml.dump(looper_config_dict, fp)
|
724
|
+
|
480
725
|
return True
|
481
726
|
|
482
727
|
|
728
|
+
def determine_pipeline_type(piface_path: str, looper_config_path: str):
|
729
|
+
"""
|
730
|
+
Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both
|
731
|
+
|
732
|
+
|
733
|
+
:param str piface_path: path to pipeline_interface
|
734
|
+
:param str looper_config_path: path to looper config file
|
735
|
+
:return Tuple[Union[str,None],Union[str,None]] : (pipeline type, resolved path) or (None, None)
|
736
|
+
"""
|
737
|
+
|
738
|
+
if piface_path is None:
|
739
|
+
return None, None
|
740
|
+
try:
|
741
|
+
piface_path = expandpath(piface_path)
|
742
|
+
except TypeError as e:
|
743
|
+
_LOGGER.warning(
|
744
|
+
f"Pipeline interface not found at given path: {piface_path}. Type Error: "
|
745
|
+
+ str(e)
|
746
|
+
)
|
747
|
+
return None, None
|
748
|
+
|
749
|
+
if not os.path.isabs(piface_path):
|
750
|
+
piface_path = os.path.realpath(
|
751
|
+
os.path.join(os.path.dirname(looper_config_path), piface_path)
|
752
|
+
)
|
753
|
+
try:
|
754
|
+
piface_dict = load_yaml(piface_path)
|
755
|
+
except FileNotFoundError:
|
756
|
+
_LOGGER.warning(f"Pipeline interface not found at given path: {piface_path}")
|
757
|
+
return None, None
|
758
|
+
|
759
|
+
pipeline_types = []
|
760
|
+
if piface_dict.get("sample_interface", None):
|
761
|
+
pipeline_types.append(PipelineLevel.SAMPLE.value)
|
762
|
+
if piface_dict.get("project_interface", None):
|
763
|
+
pipeline_types.append(PipelineLevel.PROJECT.value)
|
764
|
+
|
765
|
+
if pipeline_types == []:
|
766
|
+
raise PipelineInterfaceConfigError(
|
767
|
+
f"sample_interface and/or project_interface must be defined in each pipeline interface."
|
768
|
+
)
|
769
|
+
|
770
|
+
return pipeline_types, piface_path
|
771
|
+
|
772
|
+
|
483
773
|
def read_looper_config_file(looper_config_path: str) -> dict:
|
484
774
|
"""
|
485
775
|
Read Looper config file which includes:
|
@@ -492,19 +782,18 @@ def read_looper_config_file(looper_config_path: str) -> dict:
|
|
492
782
|
:raise MisconfigurationException: incorrect configuration.
|
493
783
|
"""
|
494
784
|
return_dict = {}
|
495
|
-
|
496
|
-
|
785
|
+
|
786
|
+
try:
|
787
|
+
with open(looper_config_path, "r") as dotfile:
|
788
|
+
dp_data = yaml.safe_load(dotfile)
|
789
|
+
except ParserError as e:
|
790
|
+
_LOGGER.warning(
|
791
|
+
"Could not load looper config file due to the following exception"
|
792
|
+
)
|
793
|
+
raise ParserError(context=str(e))
|
497
794
|
|
498
795
|
if PEP_CONFIG_KEY in dp_data:
|
499
|
-
# Looper expects the config path to live at looper.config_file
|
500
|
-
# However, user may wish to access the pep at looper.pep_config
|
501
|
-
return_dict[PEP_CONFIG_FILE_KEY] = dp_data[PEP_CONFIG_KEY]
|
502
796
|
return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
|
503
|
-
|
504
|
-
# TODO: delete it in looper 2.0
|
505
|
-
elif DOTFILE_CFG_PTH_KEY in dp_data:
|
506
|
-
return_dict[PEP_CONFIG_FILE_KEY] = dp_data[DOTFILE_CFG_PTH_KEY]
|
507
|
-
|
508
797
|
else:
|
509
798
|
raise MisconfigurationException(
|
510
799
|
f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key"
|
@@ -520,12 +809,35 @@ def read_looper_config_file(looper_config_path: str) -> dict:
|
|
520
809
|
if PIPESTAT_KEY in dp_data:
|
521
810
|
return_dict[PIPESTAT_KEY] = dp_data[PIPESTAT_KEY]
|
522
811
|
|
812
|
+
if SAMPLE_MODS_KEY in dp_data:
|
813
|
+
return_dict[SAMPLE_MODS_KEY] = dp_data[SAMPLE_MODS_KEY]
|
814
|
+
|
815
|
+
if CLI_KEY in dp_data:
|
816
|
+
return_dict[CLI_KEY] = dp_data[CLI_KEY]
|
817
|
+
|
523
818
|
if PIPELINE_INTERFACES_KEY in dp_data:
|
819
|
+
|
524
820
|
dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
821
|
+
|
822
|
+
all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY)
|
823
|
+
|
824
|
+
sample_pifaces = []
|
825
|
+
project_pifaces = []
|
826
|
+
if isinstance(all_pipeline_interfaces, str):
|
827
|
+
all_pipeline_interfaces = [all_pipeline_interfaces]
|
828
|
+
for piface in all_pipeline_interfaces:
|
829
|
+
pipeline_types, piface_path = determine_pipeline_type(
|
830
|
+
piface, looper_config_path
|
831
|
+
)
|
832
|
+
if pipeline_types is not None:
|
833
|
+
if PipelineLevel.SAMPLE.value in pipeline_types:
|
834
|
+
sample_pifaces.append(piface_path)
|
835
|
+
if PipelineLevel.PROJECT.value in pipeline_types:
|
836
|
+
project_pifaces.append(piface_path)
|
837
|
+
if len(sample_pifaces) > 0:
|
838
|
+
return_dict[SAMPLE_PL_ARG] = sample_pifaces
|
839
|
+
if len(project_pifaces) > 0:
|
840
|
+
return_dict[PROJECT_PL_ARG] = project_pifaces
|
529
841
|
|
530
842
|
else:
|
531
843
|
_LOGGER.warning(
|
@@ -537,12 +849,26 @@ def read_looper_config_file(looper_config_path: str) -> dict:
|
|
537
849
|
|
538
850
|
# Expand paths in case ENV variables are used
|
539
851
|
for k, v in return_dict.items():
|
852
|
+
if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG:
|
853
|
+
# Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474
|
854
|
+
|
855
|
+
pass
|
540
856
|
if isinstance(v, str):
|
541
857
|
v = expandpath(v)
|
542
|
-
|
543
|
-
|
544
|
-
|
858
|
+
# TODO this is messy because is_pephub_registry needs to fail on anything NOT a pephub registry path
|
859
|
+
# https://github.com/pepkit/ubiquerg/issues/43
|
860
|
+
if is_PEP_file_type(v):
|
861
|
+
if not os.path.isabs(v):
|
862
|
+
return_dict[k] = os.path.join(config_dir_path, v)
|
863
|
+
else:
|
864
|
+
return_dict[k] = v
|
865
|
+
elif is_pephub_registry_path(v):
|
545
866
|
return_dict[k] = v
|
867
|
+
else:
|
868
|
+
if not os.path.isabs(v):
|
869
|
+
return_dict[k] = os.path.join(config_dir_path, v)
|
870
|
+
else:
|
871
|
+
return_dict[k] = v
|
546
872
|
|
547
873
|
return return_dict
|
548
874
|
|
@@ -575,19 +901,23 @@ def dotfile_path(directory=os.getcwd(), must_exist=False):
|
|
575
901
|
cur_dir = parent_dir
|
576
902
|
|
577
903
|
|
578
|
-
def
|
904
|
+
def is_PEP_file_type(input_string: str) -> bool:
|
905
|
+
"""
|
906
|
+
Determines if the provided path is actually a file type that Looper can use for loading PEP
|
907
|
+
"""
|
908
|
+
|
909
|
+
PEP_FILE_TYPES = ["yaml", "csv"]
|
910
|
+
|
911
|
+
res = list(filter(input_string.endswith, PEP_FILE_TYPES)) != []
|
912
|
+
return res
|
913
|
+
|
914
|
+
|
915
|
+
def is_pephub_registry_path(input_string: str) -> bool:
|
579
916
|
"""
|
580
917
|
Check if input is a registry path to pephub
|
581
918
|
:param str input_string: path to the PEP (or registry path)
|
582
919
|
:return bool: True if input is a registry path
|
583
920
|
"""
|
584
|
-
try:
|
585
|
-
if input_string.endswith(".yaml"):
|
586
|
-
return False
|
587
|
-
except AttributeError:
|
588
|
-
raise RegistryPathException(
|
589
|
-
msg=f"Malformed registry path. Unable to parse {input_string} as a registry path."
|
590
|
-
)
|
591
921
|
try:
|
592
922
|
registry_path = RegistryPath(**parse_registry_path(input_string))
|
593
923
|
except (ValidationError, TypeError):
|
@@ -767,3 +1097,45 @@ def write_submit_script(fp, content, data):
|
|
767
1097
|
with open(fp, "w") as f:
|
768
1098
|
f.write(content)
|
769
1099
|
return fp
|
1100
|
+
|
1101
|
+
|
1102
|
+
def inspect_looper_config_file(looper_config_dict) -> None:
|
1103
|
+
"""
|
1104
|
+
Inspects looper config by printing it to terminal.
|
1105
|
+
param dict looper_config_dict: dict representing looper_config
|
1106
|
+
|
1107
|
+
"""
|
1108
|
+
# Simply print this to terminal
|
1109
|
+
print("LOOPER INSPECT")
|
1110
|
+
for key, value in looper_config_dict.items():
|
1111
|
+
print(f"{key} {value}")
|
1112
|
+
|
1113
|
+
|
1114
|
+
def expand_nested_var_templates(var_templates_dict, namespaces):
|
1115
|
+
|
1116
|
+
"Takes all var_templates as a dict and recursively expands any paths."
|
1117
|
+
|
1118
|
+
result = {}
|
1119
|
+
|
1120
|
+
for k, v in var_templates_dict.items():
|
1121
|
+
if isinstance(v, dict):
|
1122
|
+
result[k] = expand_nested_var_templates(v, namespaces)
|
1123
|
+
else:
|
1124
|
+
result[k] = expandpath(v)
|
1125
|
+
|
1126
|
+
return result
|
1127
|
+
|
1128
|
+
|
1129
|
+
def render_nested_var_templates(var_templates_dict, namespaces):
|
1130
|
+
|
1131
|
+
"Takes all var_templates as a dict and recursively renders the jinja templates."
|
1132
|
+
|
1133
|
+
result = {}
|
1134
|
+
|
1135
|
+
for k, v in var_templates_dict.items():
|
1136
|
+
if isinstance(v, dict):
|
1137
|
+
result[k] = expand_nested_var_templates(v, namespaces)
|
1138
|
+
else:
|
1139
|
+
result[k] = jinja_render_template_strictly(v, namespaces)
|
1140
|
+
|
1141
|
+
return result
|