looper 1.7.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
looper/utils.py CHANGED
@@ -1,12 +1,11 @@
1
1
  """ Helpers without an obvious logical home. """
2
2
 
3
3
  import argparse
4
- from collections import defaultdict, namedtuple
4
+ from collections import defaultdict
5
5
  import glob
6
6
  import itertools
7
7
  from logging import getLogger
8
8
  import os
9
- import sys
10
9
  from typing import *
11
10
  import re
12
11
 
@@ -14,12 +13,17 @@ import jinja2
14
13
  import yaml
15
14
  from peppy import Project as peppyProject
16
15
  from peppy.const import *
17
- from ubiquerg import convert_value, expandpath, parse_registry_path
16
+ from ubiquerg import convert_value, expandpath, parse_registry_path, deep_update
18
17
  from pephubclient.constants import RegistryPath
19
- from pydantic.error_wrappers import ValidationError
18
+ from pydantic import ValidationError
19
+ from yacman import load_yaml
20
+ from yaml.parser import ParserError
20
21
 
21
22
  from .const import *
22
- from .exceptions import MisconfigurationException, RegistryPathException
23
+ from .command_models.commands import SUPPORTED_COMMANDS
24
+ from .exceptions import MisconfigurationException, PipelineInterfaceConfigError
25
+ from rich.console import Console
26
+ from rich.pretty import pprint
23
27
 
24
28
  _LOGGER = getLogger(__name__)
25
29
 
@@ -94,7 +98,9 @@ def fetch_sample_flags(prj, sample, pl_name, flag_dir=None):
94
98
  return [
95
99
  x
96
100
  for x in folder_contents
97
- if os.path.splitext(x)[1] == ".flag" and os.path.basename(x).startswith(pl_name)
101
+ if os.path.splitext(x)[1] == ".flag"
102
+ and os.path.basename(x).startswith(pl_name)
103
+ and sample.sample_name in x
98
104
  ]
99
105
 
100
106
 
@@ -250,22 +256,54 @@ def read_yaml_file(filepath):
250
256
  return data
251
257
 
252
258
 
253
- def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
259
+ def enrich_args_via_cfg(
260
+ subcommand_name,
261
+ parser_args,
262
+ aux_parser,
263
+ test_args=None,
264
+ cli_modifiers=None,
265
+ ):
254
266
  """
255
- Read in a looper dotfile and set arguments.
267
+ Read in a looper dotfile, pep config and set arguments.
256
268
 
257
- Priority order: CLI > dotfile/config > parser default
269
+ Priority order: CLI > dotfile/config > pep_config > parser default
258
270
 
271
+ :param subcommand name: the name of the command used
259
272
  :param argparse.Namespace parser_args: parsed args by the original parser
260
- :param argparse.Namespace aux_parser: parsed args by the a parser
273
+ :param argparse.Namespace aux_parser: parsed args by the argument parser
261
274
  with defaults suppressed
275
+ :param dict test_args: dict of args used for pytesting
276
+ :param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file
262
277
  :return argparse.Namespace: selected argument values
263
278
  """
279
+
280
+ # Did the user provide arguments in the PEP config?
264
281
  cfg_args_all = (
265
- _get_subcommand_args(parser_args)
266
- if os.path.exists(parser_args.config_file)
282
+ _get_subcommand_args(subcommand_name, parser_args)
283
+ if os.path.exists(parser_args.pep_config)
267
284
  else dict()
268
285
  )
286
+ if not cfg_args_all:
287
+ cfg_args_all = {}
288
+
289
+ # Did the user provide arguments/modifiers in the looper config file?
290
+ looper_config_cli_modifiers = None
291
+ if cli_modifiers:
292
+ if str(subcommand_name) in cli_modifiers:
293
+ looper_config_cli_modifiers = cli_modifiers[subcommand_name]
294
+ looper_config_cli_modifiers = (
295
+ {k.replace("-", "_"): v for k, v in looper_config_cli_modifiers.items()}
296
+ if looper_config_cli_modifiers
297
+ else None
298
+ )
299
+
300
+ if looper_config_cli_modifiers:
301
+ _LOGGER.warning(
302
+ "CLI modifiers were provided in Looper Config and in PEP Project Config. Merging..."
303
+ )
304
+ deep_update(cfg_args_all, looper_config_cli_modifiers)
305
+ _LOGGER.debug(msg=f"Merged CLI modifiers: {cfg_args_all}")
306
+
269
307
  result = argparse.Namespace()
270
308
  if test_args:
271
309
  cli_args, _ = aux_parser.parse_known_args(args=test_args)
@@ -273,23 +311,51 @@ def enrich_args_via_cfg(parser_args, aux_parser, test_args=None):
273
311
  else:
274
312
  cli_args, _ = aux_parser.parse_known_args()
275
313
 
276
- for dest in vars(parser_args):
277
- if dest not in POSITIONAL or not hasattr(result, dest):
278
- if dest in cli_args:
279
- x = getattr(cli_args, dest)
280
- r = convert_value(x) if isinstance(x, str) else x
281
- elif cfg_args_all is not None and dest in cfg_args_all:
282
- if isinstance(cfg_args_all[dest], list):
283
- r = [convert_value(i) for i in cfg_args_all[dest]]
314
+ # If any CLI args were provided, make sure they take priority
315
+ if cli_args:
316
+ r = getattr(cli_args, subcommand_name)
317
+ for k, v in cfg_args_all.items():
318
+ if k in r:
319
+ cfg_args_all[k] = getattr(r, k)
320
+
321
+ def set_single_arg(argname, default_source_namespace, result_namespace):
322
+ if argname not in POSITIONAL or not hasattr(result, argname):
323
+ if argname in cli_args:
324
+ cli_provided_value = getattr(cli_args, argname)
325
+ r = (
326
+ convert_value(cli_provided_value)
327
+ if isinstance(cli_provided_value, str)
328
+ else cli_provided_value
329
+ )
330
+ elif cfg_args_all is not None and argname in cfg_args_all:
331
+ if isinstance(cfg_args_all[argname], list):
332
+ r = [convert_value(i) for i in cfg_args_all[argname]]
333
+ elif isinstance(cfg_args_all[argname], dict):
334
+ r = cfg_args_all[argname]
284
335
  else:
285
- r = convert_value(cfg_args_all[dest])
336
+ r = convert_value(cfg_args_all[argname])
286
337
  else:
287
- r = getattr(parser_args, dest)
288
- setattr(result, dest, r)
338
+ r = getattr(default_source_namespace, argname)
339
+ setattr(result_namespace, argname, r)
340
+
341
+ for top_level_argname in vars(parser_args):
342
+ if top_level_argname not in [cmd.name for cmd in SUPPORTED_COMMANDS]:
343
+ # this argument is a top-level argument
344
+ set_single_arg(top_level_argname, parser_args, result)
345
+ else:
346
+ # this argument actually is a subcommand
347
+ enriched_command_namespace = argparse.Namespace()
348
+ command_namespace = getattr(parser_args, top_level_argname)
349
+ if command_namespace:
350
+ for argname in vars(command_namespace):
351
+ set_single_arg(
352
+ argname, command_namespace, enriched_command_namespace
353
+ )
354
+ setattr(result, top_level_argname, enriched_command_namespace)
289
355
  return result
290
356
 
291
357
 
292
- def _get_subcommand_args(parser_args):
358
+ def _get_subcommand_args(subcommand_name, parser_args):
293
359
  """
294
360
  Get the union of values for the subcommand arguments from
295
361
  Project.looper, Project.looper.cli.<subcommand> and Project.looper.cli.all.
@@ -304,7 +370,7 @@ def _get_subcommand_args(parser_args):
304
370
  """
305
371
  args = dict()
306
372
  cfg = peppyProject(
307
- parser_args.config_file,
373
+ parser_args.pep_config,
308
374
  defer_samples_creation=True,
309
375
  amendments=parser_args.amend,
310
376
  )
@@ -321,8 +387,8 @@ def _get_subcommand_args(parser_args):
321
387
  else dict()
322
388
  )
323
389
  args.update(
324
- cfg_args[parser_args.command] or dict()
325
- if parser_args.command in cfg_args
390
+ cfg_args[subcommand_name] or dict()
391
+ if subcommand_name in cfg_args
326
392
  else dict()
327
393
  )
328
394
  except (TypeError, KeyError, AttributeError, ValueError) as e:
@@ -346,40 +412,65 @@ def _get_subcommand_args(parser_args):
346
412
  return args
347
413
 
348
414
 
349
- def init_generic_pipeline():
415
+ def init_generic_pipeline(pipelinepath: Optional[str] = None):
350
416
  """
351
417
  Create generic pipeline interface
352
418
  """
353
- try:
354
- os.makedirs("pipeline")
355
- except FileExistsError:
356
- pass
419
+ console = Console()
357
420
 
358
421
  # Destination one level down from CWD in pipeline folder
359
- dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
422
+ if not pipelinepath:
423
+ try:
424
+ os.makedirs("pipeline")
425
+ except FileExistsError:
426
+ pass
427
+
428
+ dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_PIPELINE)
429
+ else:
430
+ if os.path.isabs(pipelinepath):
431
+ dest_file = pipelinepath
432
+ else:
433
+ dest_file = os.path.join(os.getcwd(), os.path.relpath(pipelinepath))
434
+ try:
435
+ os.makedirs(os.path.dirname(dest_file))
436
+ except FileExistsError:
437
+ pass
360
438
 
361
439
  # Create Generic Pipeline Interface
362
440
  generic_pipeline_dict = {
363
441
  "pipeline_name": "default_pipeline_name",
364
- "pipeline_type": "sample",
365
442
  "output_schema": "output_schema.yaml",
366
- "var_templates": {"pipeline": "{looper.piface_dir}/pipeline.sh"},
367
- "command_template": "{pipeline.var_templates.pipeline} {sample.file} "
368
- "--output-parent {looper.sample_output_folder}",
443
+ "sample_interface": {
444
+ "command_template": "{looper.piface_dir}/count_lines.sh {sample.file} "
445
+ "--output-parent {looper.sample_output_folder}"
446
+ },
369
447
  }
370
448
 
449
+ console.rule(f"\n[magenta]Pipeline Interface[/magenta]")
371
450
  # Write file
372
451
  if not os.path.exists(dest_file):
452
+ pprint(generic_pipeline_dict, expand_all=True)
453
+
373
454
  with open(dest_file, "w") as file:
374
455
  yaml.dump(generic_pipeline_dict, file)
375
- print(f"Pipeline interface successfully created at: {dest_file}")
456
+
457
+ console.print(
458
+ f"Pipeline interface successfully created at: [yellow]{dest_file}[/yellow]"
459
+ )
460
+
376
461
  else:
377
- print(
378
- f"Pipeline interface file already exists `{dest_file}`. Skipping creation.."
462
+ console.print(
463
+ f"Pipeline interface file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
379
464
  )
380
465
 
381
466
  # Create Generic Output Schema
382
- dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
467
+ if not pipelinepath:
468
+ dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_OUTPUT_SCHEMA)
469
+ else:
470
+ dest_file = os.path.join(
471
+ os.path.dirname(dest_file), LOOPER_GENERIC_OUTPUT_SCHEMA
472
+ )
473
+
383
474
  generic_output_schema_dict = {
384
475
  "pipeline_name": "default_pipeline_name",
385
476
  "samples": {
@@ -389,27 +480,45 @@ def init_generic_pipeline():
389
480
  }
390
481
  },
391
482
  }
483
+
484
+ console.rule(f"\n[magenta]Output Schema[/magenta]")
392
485
  # Write file
393
486
  if not os.path.exists(dest_file):
487
+ pprint(generic_output_schema_dict, expand_all=True)
394
488
  with open(dest_file, "w") as file:
395
489
  yaml.dump(generic_output_schema_dict, file)
396
- print(f"Output schema successfully created at: {dest_file}")
490
+ console.print(
491
+ f"Output schema successfully created at: [yellow]{dest_file}[/yellow]"
492
+ )
397
493
  else:
398
- print(f"Output schema file already exists `{dest_file}`. Skipping creation..")
494
+ console.print(
495
+ f"Output schema file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
496
+ )
399
497
 
498
+ console.rule(f"\n[magenta]Example Pipeline Shell Script[/magenta]")
400
499
  # Create Generic countlines.sh
401
- dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
500
+
501
+ if not pipelinepath:
502
+ dest_file = os.path.join(os.getcwd(), "pipeline", LOOPER_GENERIC_COUNT_LINES)
503
+ else:
504
+ dest_file = os.path.join(os.path.dirname(dest_file), LOOPER_GENERIC_COUNT_LINES)
505
+
402
506
  shell_code = """#!/bin/bash
403
507
  linecount=`wc -l $1 | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '`
404
508
  pipestat report -r $2 -i 'number_of_lines' -v $linecount -c $3
405
509
  echo "Number of lines: $linecount"
406
510
  """
407
511
  if not os.path.exists(dest_file):
512
+ console.print(shell_code)
408
513
  with open(dest_file, "w") as file:
409
514
  file.write(shell_code)
410
- print(f"count_lines.sh successfully created at: {dest_file}")
515
+ console.print(
516
+ f"count_lines.sh successfully created at: [yellow]{dest_file}[/yellow]"
517
+ )
411
518
  else:
412
- print(f"count_lines.sh file already exists `{dest_file}`. Skipping creation..")
519
+ console.print(
520
+ f"count_lines.sh file already exists [yellow]`{dest_file}`[/yellow]. Skipping creation.."
521
+ )
413
522
 
414
523
  return True
415
524
 
@@ -444,12 +553,18 @@ def initiate_looper_config(
444
553
  :param bool force: whether the existing file should be overwritten
445
554
  :return bool: whether the file was initialized
446
555
  """
556
+ console = Console()
557
+ console.clear()
558
+ console.rule(f"\n[magenta]Looper initialization[/magenta]")
559
+
447
560
  if os.path.exists(looper_config_path) and not force:
448
- print(f"Can't initialize, file exists: {looper_config_path}")
561
+ console.print(
562
+ f"[red]Can't initialize, file exists:[/red] [yellow]{looper_config_path}[/yellow]"
563
+ )
449
564
  return False
450
565
 
451
566
  if pep_path:
452
- if is_registry_path(pep_path):
567
+ if is_pephub_registry_path(pep_path):
453
568
  pass
454
569
  else:
455
570
  pep_path = expandpath(pep_path)
@@ -465,21 +580,196 @@ def initiate_looper_config(
465
580
  if not output_dir:
466
581
  output_dir = "."
467
582
 
583
+ if sample_pipeline_interfaces is None or sample_pipeline_interfaces == []:
584
+ sample_pipeline_interfaces = "pipeline_interface1.yaml"
585
+
586
+ if project_pipeline_interfaces is None or project_pipeline_interfaces == []:
587
+ project_pipeline_interfaces = "pipeline_interface2.yaml"
588
+
468
589
  looper_config_dict = {
469
590
  "pep_config": os.path.relpath(pep_path),
470
591
  "output_dir": output_dir,
471
- "pipeline_interfaces": {
472
- "sample": sample_pipeline_interfaces,
473
- "project": project_pipeline_interfaces,
474
- },
592
+ "pipeline_interfaces": [
593
+ sample_pipeline_interfaces,
594
+ project_pipeline_interfaces,
595
+ ],
475
596
  }
476
597
 
598
+ pprint(looper_config_dict, expand_all=True)
599
+
477
600
  with open(looper_config_path, "w") as dotfile:
478
601
  yaml.dump(looper_config_dict, dotfile)
479
- print(f"Initialized looper config file: {looper_config_path}")
602
+ console.print(
603
+ f"Initialized looper config file: [yellow]{looper_config_path}[/yellow]"
604
+ )
605
+
606
+ return True
607
+
608
+
609
+ def looper_config_tutorial():
610
+ """
611
+ Prompt a user through configuring a .looper.yaml file for a new project.
612
+
613
+ :return bool: whether the file was initialized
614
+ """
615
+
616
+ console = Console()
617
+ console.clear()
618
+ console.rule(f"\n[magenta]Looper initialization[/magenta]")
619
+
620
+ looper_cfg_path = ".looper.yaml" # not changeable
621
+
622
+ if os.path.exists(looper_cfg_path):
623
+ console.print(
624
+ f"[bold red]File exists at '{looper_cfg_path}'. Delete it to re-initialize. \n[/bold red]"
625
+ )
626
+ raise SystemExit
627
+
628
+ cfg = {}
629
+
630
+ console.print(
631
+ "This utility will walk you through creating a [yellow].looper.yaml[/yellow] file."
632
+ )
633
+ console.print("See [yellow]`looper init --help`[/yellow] for details.")
634
+ console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
635
+ console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
636
+
637
+ DEFAULTS = { # What you get if you just press enter
638
+ "pep_config": "databio/example",
639
+ "output_dir": "results",
640
+ "piface_path": "pipeline/pipeline_interface.yaml",
641
+ "project_name": os.path.basename(os.getcwd()),
642
+ }
643
+
644
+ cfg["project_name"] = (
645
+ console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >")
646
+ or DEFAULTS["project_name"]
647
+ )
648
+
649
+ cfg["pep_config"] = (
650
+ console.input(
651
+ f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
652
+ )
653
+ or DEFAULTS["pep_config"]
654
+ )
655
+
656
+ if not os.path.exists(cfg["pep_config"]) and not is_pephub_registry_path(
657
+ cfg["pep_config"]
658
+ ):
659
+ console.print(
660
+ f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
661
+ )
662
+
663
+ cfg["output_dir"] = (
664
+ console.input(
665
+ f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
666
+ )
667
+ or DEFAULTS["output_dir"]
668
+ )
669
+
670
+ add_more_pifaces = True
671
+ piface_paths = []
672
+ while add_more_pifaces:
673
+ piface_path = (
674
+ console.input(
675
+ "Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
676
+ )
677
+ or None
678
+ )
679
+ if piface_path is None:
680
+ if piface_paths == []:
681
+ piface_paths.append(DEFAULTS["piface_path"])
682
+ add_more_pifaces = False
683
+ else:
684
+ piface_paths.append(piface_path)
685
+
686
+ console.print("\n")
687
+
688
+ console.print(
689
+ f"""\
690
+ [yellow]pep_config:[/yellow] {cfg['pep_config']}
691
+ [yellow]output_dir:[/yellow] {cfg['output_dir']}
692
+ [yellow]pipeline_interfaces:[/yellow]
693
+ - {piface_paths}
694
+ """
695
+ )
696
+
697
+ for piface_path in piface_paths:
698
+ if not os.path.exists(piface_path):
699
+ console.print(
700
+ f"[bold red]Warning:[/bold red] File does not exist at [yellow]{piface_path}[/yellow]"
701
+ )
702
+ console.print(
703
+ "Do you wish to initialize a generic pipeline interface? [bold green]Y[/bold green]/[red]n[/red]..."
704
+ )
705
+ selection = None
706
+ while selection not in ["y", "n"]:
707
+ selection = console.input("\nSelection: ").lower().strip()
708
+ if selection == "n":
709
+ console.print(
710
+ "Use command [yellow]`looper init_piface`[/yellow] to create a generic pipeline interface."
711
+ )
712
+ if selection == "y":
713
+ init_generic_pipeline(pipelinepath=piface_path)
714
+
715
+ console.print(f"Writing config file to [yellow]{looper_cfg_path}[/yellow]")
716
+
717
+ looper_config_dict = {}
718
+ looper_config_dict["pep_config"] = cfg["pep_config"]
719
+ looper_config_dict["output_dir"] = cfg["output_dir"]
720
+ looper_config_dict["pipeline_interfaces"] = piface_paths
721
+
722
+ with open(looper_cfg_path, "w") as fp:
723
+ yaml.dump(looper_config_dict, fp)
724
+
480
725
  return True
481
726
 
482
727
 
728
+ def determine_pipeline_type(piface_path: str, looper_config_path: str):
729
+ """
730
+ Read pipeline interface from disk and determine if it contains "sample_interface", "project_interface" or both
731
+
732
+
733
+ :param str piface_path: path to pipeline_interface
734
+ :param str looper_config_path: path to looper config file
735
+ :return Tuple[Union[str,None],Union[str,None]] : (pipeline type, resolved path) or (None, None)
736
+ """
737
+
738
+ if piface_path is None:
739
+ return None, None
740
+ try:
741
+ piface_path = expandpath(piface_path)
742
+ except TypeError as e:
743
+ _LOGGER.warning(
744
+ f"Pipeline interface not found at given path: {piface_path}. Type Error: "
745
+ + str(e)
746
+ )
747
+ return None, None
748
+
749
+ if not os.path.isabs(piface_path):
750
+ piface_path = os.path.realpath(
751
+ os.path.join(os.path.dirname(looper_config_path), piface_path)
752
+ )
753
+ try:
754
+ piface_dict = load_yaml(piface_path)
755
+ except FileNotFoundError:
756
+ _LOGGER.warning(f"Pipeline interface not found at given path: {piface_path}")
757
+ return None, None
758
+
759
+ pipeline_types = []
760
+ if piface_dict.get("sample_interface", None):
761
+ pipeline_types.append(PipelineLevel.SAMPLE.value)
762
+ if piface_dict.get("project_interface", None):
763
+ pipeline_types.append(PipelineLevel.PROJECT.value)
764
+
765
+ if pipeline_types == []:
766
+ raise PipelineInterfaceConfigError(
767
+ f"sample_interface and/or project_interface must be defined in each pipeline interface."
768
+ )
769
+
770
+ return pipeline_types, piface_path
771
+
772
+
483
773
  def read_looper_config_file(looper_config_path: str) -> dict:
484
774
  """
485
775
  Read Looper config file which includes:
@@ -492,19 +782,18 @@ def read_looper_config_file(looper_config_path: str) -> dict:
492
782
  :raise MisconfigurationException: incorrect configuration.
493
783
  """
494
784
  return_dict = {}
495
- with open(looper_config_path, "r") as dotfile:
496
- dp_data = yaml.safe_load(dotfile)
785
+
786
+ try:
787
+ with open(looper_config_path, "r") as dotfile:
788
+ dp_data = yaml.safe_load(dotfile)
789
+ except ParserError as e:
790
+ _LOGGER.warning(
791
+ "Could not load looper config file due to the following exception"
792
+ )
793
+ raise ParserError(context=str(e))
497
794
 
498
795
  if PEP_CONFIG_KEY in dp_data:
499
- # Looper expects the config path to live at looper.config_file
500
- # However, user may wish to access the pep at looper.pep_config
501
- return_dict[PEP_CONFIG_FILE_KEY] = dp_data[PEP_CONFIG_KEY]
502
796
  return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
503
-
504
- # TODO: delete it in looper 2.0
505
- elif DOTFILE_CFG_PTH_KEY in dp_data:
506
- return_dict[PEP_CONFIG_FILE_KEY] = dp_data[DOTFILE_CFG_PTH_KEY]
507
-
508
797
  else:
509
798
  raise MisconfigurationException(
510
799
  f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key"
@@ -520,12 +809,35 @@ def read_looper_config_file(looper_config_path: str) -> dict:
520
809
  if PIPESTAT_KEY in dp_data:
521
810
  return_dict[PIPESTAT_KEY] = dp_data[PIPESTAT_KEY]
522
811
 
812
+ if SAMPLE_MODS_KEY in dp_data:
813
+ return_dict[SAMPLE_MODS_KEY] = dp_data[SAMPLE_MODS_KEY]
814
+
815
+ if CLI_KEY in dp_data:
816
+ return_dict[CLI_KEY] = dp_data[CLI_KEY]
817
+
523
818
  if PIPELINE_INTERFACES_KEY in dp_data:
819
+
524
820
  dp_data.setdefault(PIPELINE_INTERFACES_KEY, {})
525
- return_dict[SAMPLE_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get("sample")
526
- return_dict[PROJECT_PL_ARG] = dp_data.get(PIPELINE_INTERFACES_KEY).get(
527
- "project"
528
- )
821
+
822
+ all_pipeline_interfaces = dp_data.get(PIPELINE_INTERFACES_KEY)
823
+
824
+ sample_pifaces = []
825
+ project_pifaces = []
826
+ if isinstance(all_pipeline_interfaces, str):
827
+ all_pipeline_interfaces = [all_pipeline_interfaces]
828
+ for piface in all_pipeline_interfaces:
829
+ pipeline_types, piface_path = determine_pipeline_type(
830
+ piface, looper_config_path
831
+ )
832
+ if pipeline_types is not None:
833
+ if PipelineLevel.SAMPLE.value in pipeline_types:
834
+ sample_pifaces.append(piface_path)
835
+ if PipelineLevel.PROJECT.value in pipeline_types:
836
+ project_pifaces.append(piface_path)
837
+ if len(sample_pifaces) > 0:
838
+ return_dict[SAMPLE_PL_ARG] = sample_pifaces
839
+ if len(project_pifaces) > 0:
840
+ return_dict[PROJECT_PL_ARG] = project_pifaces
529
841
 
530
842
  else:
531
843
  _LOGGER.warning(
@@ -537,12 +849,26 @@ def read_looper_config_file(looper_config_path: str) -> dict:
537
849
 
538
850
  # Expand paths in case ENV variables are used
539
851
  for k, v in return_dict.items():
852
+ if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG:
853
+ # Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474
854
+
855
+ pass
540
856
  if isinstance(v, str):
541
857
  v = expandpath(v)
542
- if not os.path.isabs(v) and not is_registry_path(v):
543
- return_dict[k] = os.path.join(config_dir_path, v)
544
- else:
858
+ # TODO this is messy because is_pephub_registry needs to fail on anything NOT a pephub registry path
859
+ # https://github.com/pepkit/ubiquerg/issues/43
860
+ if is_PEP_file_type(v):
861
+ if not os.path.isabs(v):
862
+ return_dict[k] = os.path.join(config_dir_path, v)
863
+ else:
864
+ return_dict[k] = v
865
+ elif is_pephub_registry_path(v):
545
866
  return_dict[k] = v
867
+ else:
868
+ if not os.path.isabs(v):
869
+ return_dict[k] = os.path.join(config_dir_path, v)
870
+ else:
871
+ return_dict[k] = v
546
872
 
547
873
  return return_dict
548
874
 
@@ -575,19 +901,23 @@ def dotfile_path(directory=os.getcwd(), must_exist=False):
575
901
  cur_dir = parent_dir
576
902
 
577
903
 
578
- def is_registry_path(input_string: str) -> bool:
904
+ def is_PEP_file_type(input_string: str) -> bool:
905
+ """
906
+ Determines if the provided path is actually a file type that Looper can use for loading PEP
907
+ """
908
+
909
+ PEP_FILE_TYPES = ["yaml", "csv"]
910
+
911
+ res = list(filter(input_string.endswith, PEP_FILE_TYPES)) != []
912
+ return res
913
+
914
+
915
+ def is_pephub_registry_path(input_string: str) -> bool:
579
916
  """
580
917
  Check if input is a registry path to pephub
581
918
  :param str input_string: path to the PEP (or registry path)
582
919
  :return bool: True if input is a registry path
583
920
  """
584
- try:
585
- if input_string.endswith(".yaml"):
586
- return False
587
- except AttributeError:
588
- raise RegistryPathException(
589
- msg=f"Malformed registry path. Unable to parse {input_string} as a registry path."
590
- )
591
921
  try:
592
922
  registry_path = RegistryPath(**parse_registry_path(input_string))
593
923
  except (ValidationError, TypeError):
@@ -767,3 +1097,45 @@ def write_submit_script(fp, content, data):
767
1097
  with open(fp, "w") as f:
768
1098
  f.write(content)
769
1099
  return fp
1100
+
1101
+
1102
+ def inspect_looper_config_file(looper_config_dict) -> None:
1103
+ """
1104
+ Inspects looper config by printing it to terminal.
1105
+ param dict looper_config_dict: dict representing looper_config
1106
+
1107
+ """
1108
+ # Simply print this to terminal
1109
+ print("LOOPER INSPECT")
1110
+ for key, value in looper_config_dict.items():
1111
+ print(f"{key} {value}")
1112
+
1113
+
1114
+ def expand_nested_var_templates(var_templates_dict, namespaces):
1115
+
1116
+ "Takes all var_templates as a dict and recursively expands any paths."
1117
+
1118
+ result = {}
1119
+
1120
+ for k, v in var_templates_dict.items():
1121
+ if isinstance(v, dict):
1122
+ result[k] = expand_nested_var_templates(v, namespaces)
1123
+ else:
1124
+ result[k] = expandpath(v)
1125
+
1126
+ return result
1127
+
1128
+
1129
+ def render_nested_var_templates(var_templates_dict, namespaces):
1130
+
1131
+ "Takes all var_templates as a dict and recursively renders the jinja templates."
1132
+
1133
+ result = {}
1134
+
1135
+ for k, v in var_templates_dict.items():
1136
+ if isinstance(v, dict):
1137
+ result[k] = expand_nested_var_templates(v, namespaces)
1138
+ else:
1139
+ result[k] = jinja_render_template_strictly(v, namespaces)
1140
+
1141
+ return result