looper 2.0.0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/_version.py +1 -1
- looper/cli_divvy.py +6 -10
- looper/cli_pydantic.py +33 -40
- looper/command_models/arguments.py +0 -15
- looper/command_models/commands.py +1 -5
- looper/conductor.py +0 -1
- looper/divvy.py +9 -26
- looper/exceptions.py +1 -9
- looper/looper.py +16 -50
- looper/utils.py +76 -82
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/METADATA +15 -23
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/RECORD +16 -16
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/WHEEL +1 -1
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/entry_points.txt +1 -1
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/LICENSE.txt +0 -0
- {looper-2.0.0.dist-info → looper-2.0.0a1.dist-info}/top_level.txt +0 -0
looper/_version.py
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
__version__ = "2.0.
|
1
|
+
__version__ = "2.0.0a1"
|
2
2
|
# You must change the version in parser = pydantic_argparse.ArgumentParser in cli_pydantic.py!!!
|
looper/cli_divvy.py
CHANGED
@@ -53,10 +53,10 @@ def build_argparser():
|
|
53
53
|
|
54
54
|
for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
|
55
55
|
sp.add_argument(
|
56
|
-
"
|
56
|
+
"config", nargs="?", default=None, help="Divvy configuration file."
|
57
57
|
)
|
58
58
|
|
59
|
-
sps["init"].add_argument("
|
59
|
+
sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
|
60
60
|
|
61
61
|
for sp in [sps["inspect"]]:
|
62
62
|
sp.add_argument(
|
@@ -124,11 +124,9 @@ def main():
|
|
124
124
|
sys.exit(0)
|
125
125
|
|
126
126
|
_LOGGER.debug("Divvy config: {}".format(args.config))
|
127
|
-
|
128
127
|
divcfg = select_divvy_config(args.config)
|
129
|
-
|
130
128
|
_LOGGER.info("Using divvy config: {}".format(divcfg))
|
131
|
-
dcc = ComputingConfiguration
|
129
|
+
dcc = ComputingConfiguration(filepath=divcfg)
|
132
130
|
|
133
131
|
if args.command == "list":
|
134
132
|
# Output header via logger and content via print so the user can
|
@@ -144,13 +142,11 @@ def main():
|
|
144
142
|
for pkg_name, pkg in dcc.compute_packages.items():
|
145
143
|
if pkg_name == args.package:
|
146
144
|
found = True
|
147
|
-
with open(pkg
|
145
|
+
with open(pkg.submission_template, "r") as f:
|
148
146
|
print(f.read())
|
149
|
-
_LOGGER.info(
|
150
|
-
"Submission command is: " + pkg["submission_command"] + "\n"
|
151
|
-
)
|
147
|
+
_LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
|
152
148
|
if pkg_name == "docker":
|
153
|
-
print("Docker args are: " + pkg
|
149
|
+
print("Docker args are: " + pkg.docker_args)
|
154
150
|
|
155
151
|
if not found:
|
156
152
|
_LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
|
looper/cli_pydantic.py
CHANGED
@@ -26,6 +26,8 @@ from eido import inspect_project
|
|
26
26
|
from pephubclient import PEPHubClient
|
27
27
|
from pydantic_argparse.argparse.parser import ArgumentParser
|
28
28
|
|
29
|
+
from divvy import select_divvy_config
|
30
|
+
|
29
31
|
from . import __version__
|
30
32
|
|
31
33
|
from .command_models.arguments import ArgumentEnum
|
@@ -126,8 +128,15 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
|
|
126
128
|
console = Console()
|
127
129
|
console.clear()
|
128
130
|
console.rule(f"\n[magenta]Looper initialization[/magenta]")
|
129
|
-
|
130
|
-
|
131
|
+
console.print(
|
132
|
+
"[bold]Would you like to follow a guided tutorial?[/bold] [green]Y[/green] / [red]n[/red]..."
|
133
|
+
)
|
134
|
+
|
135
|
+
selection = None
|
136
|
+
while selection not in ["y", "n"]:
|
137
|
+
selection = console.input("\nSelection: ").lower().strip()
|
138
|
+
|
139
|
+
if selection == "n":
|
131
140
|
console.clear()
|
132
141
|
return int(
|
133
142
|
not initiate_looper_config(
|
@@ -163,14 +172,11 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
|
|
163
172
|
else:
|
164
173
|
setattr(subcommand_args, looper_config_key, looper_config_item)
|
165
174
|
|
166
|
-
except OSError
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
else:
|
172
|
-
_LOGGER.warning(e)
|
173
|
-
|
175
|
+
except OSError:
|
176
|
+
parser.print_help(sys.stderr)
|
177
|
+
_LOGGER.warning(
|
178
|
+
f"Looper config file does not exist. Use looper init to create one at {looper_cfg_path}."
|
179
|
+
)
|
174
180
|
sys.exit(1)
|
175
181
|
|
176
182
|
subcommand_args = enrich_args_via_cfg(
|
@@ -219,7 +225,7 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
|
|
219
225
|
amendments=subcommand_args.amend,
|
220
226
|
divcfg_path=divcfg,
|
221
227
|
runp=subcommand_name == "runp",
|
222
|
-
project_dict=PEPHubClient().
|
228
|
+
project_dict=PEPHubClient()._load_raw_pep(
|
223
229
|
registry_path=subcommand_args.pep_config
|
224
230
|
),
|
225
231
|
**{
|
@@ -334,13 +340,13 @@ def run_looper(args: TopLevelParser, parser: ArgumentParser, test_args=None):
|
|
334
340
|
_LOGGER.warning("No looper configuration was supplied.")
|
335
341
|
|
336
342
|
|
337
|
-
def main(test_args=None) ->
|
343
|
+
def main(test_args=None) -> None:
|
338
344
|
parser = pydantic_argparse.ArgumentParser(
|
339
345
|
model=TopLevelParser,
|
340
346
|
prog="looper",
|
341
347
|
description="Looper: A job submitter for Portable Encapsulated Projects",
|
342
348
|
add_help=True,
|
343
|
-
version="2.0.
|
349
|
+
version="2.0.0a1",
|
344
350
|
)
|
345
351
|
|
346
352
|
parser = add_short_arguments(parser, ArgumentEnum)
|
@@ -353,10 +359,6 @@ def main(test_args=None) -> dict:
|
|
353
359
|
return run_looper(args, parser, test_args=test_args)
|
354
360
|
|
355
361
|
|
356
|
-
def main_cli() -> None:
|
357
|
-
main()
|
358
|
-
|
359
|
-
|
360
362
|
def _proc_resources_spec(args):
|
361
363
|
"""
|
362
364
|
Process CLI-sources compute setting specification. There are two sources
|
@@ -383,29 +385,20 @@ def _proc_resources_spec(args):
|
|
383
385
|
settings_data = {}
|
384
386
|
if not spec:
|
385
387
|
return settings_data
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
if bads:
|
401
|
-
raise ValueError(
|
402
|
-
"Could not correctly parse itemized compute specification. "
|
403
|
-
"Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
|
404
|
-
)
|
405
|
-
elif isinstance(spec, dict):
|
406
|
-
for key, value in spec.items():
|
407
|
-
settings_data[key] = value
|
408
|
-
|
388
|
+
pairs = [(kv, kv.split("=")) for kv in spec]
|
389
|
+
bads = []
|
390
|
+
for orig, pair in pairs:
|
391
|
+
try:
|
392
|
+
k, v = pair
|
393
|
+
except ValueError:
|
394
|
+
bads.append(orig)
|
395
|
+
else:
|
396
|
+
settings_data[k] = v
|
397
|
+
if bads:
|
398
|
+
raise ValueError(
|
399
|
+
"Could not correctly parse itemized compute specification. "
|
400
|
+
"Correct format: " + EXAMPLE_COMPUTE_SPEC_FMT
|
401
|
+
)
|
409
402
|
return settings_data
|
410
403
|
|
411
404
|
|
@@ -184,20 +184,6 @@ class ArgumentEnum(enum.Enum):
|
|
184
184
|
default=(str, None),
|
185
185
|
description="Output directory",
|
186
186
|
)
|
187
|
-
REPORT_OUTPUT_DIR = Argument(
|
188
|
-
name="report_dir",
|
189
|
-
alias="-r",
|
190
|
-
default=(str, None),
|
191
|
-
description="Set location for looper report and looper table outputs",
|
192
|
-
)
|
193
|
-
|
194
|
-
GENERIC = Argument(
|
195
|
-
name="generic",
|
196
|
-
alias="-g",
|
197
|
-
default=(bool, False),
|
198
|
-
description="Use generic looper config?",
|
199
|
-
)
|
200
|
-
|
201
187
|
SAMPLE_PIPELINE_INTERFACES = Argument(
|
202
188
|
name="sample_pipeline_interfaces",
|
203
189
|
alias="-S",
|
@@ -242,7 +228,6 @@ class ArgumentEnum(enum.Enum):
|
|
242
228
|
)
|
243
229
|
PACKAGE = Argument(
|
244
230
|
name="package",
|
245
|
-
alias="-p",
|
246
231
|
default=(str, None),
|
247
232
|
description="Name of computing resource package to use",
|
248
233
|
)
|
@@ -124,9 +124,7 @@ RunProjectParser = Command(
|
|
124
124
|
TableParser = Command(
|
125
125
|
"table",
|
126
126
|
MESSAGE_BY_SUBCOMMAND["table"],
|
127
|
-
[
|
128
|
-
ArgumentEnum.REPORT_OUTPUT_DIR.value,
|
129
|
-
],
|
127
|
+
[],
|
130
128
|
)
|
131
129
|
|
132
130
|
|
@@ -136,7 +134,6 @@ ReportParser = Command(
|
|
136
134
|
MESSAGE_BY_SUBCOMMAND["report"],
|
137
135
|
[
|
138
136
|
ArgumentEnum.PORTABLE.value,
|
139
|
-
ArgumentEnum.REPORT_OUTPUT_DIR.value,
|
140
137
|
],
|
141
138
|
)
|
142
139
|
|
@@ -190,7 +187,6 @@ InitParser = Command(
|
|
190
187
|
ArgumentEnum.PEP_CONFIG.value,
|
191
188
|
ArgumentEnum.SAMPLE_PIPELINE_INTERFACES.value,
|
192
189
|
ArgumentEnum.PROJECT_PIPELINE_INTERFACES.value,
|
193
|
-
ArgumentEnum.GENERIC.value,
|
194
190
|
],
|
195
191
|
)
|
196
192
|
|
looper/conductor.py
CHANGED
@@ -661,7 +661,6 @@ class SubmissionConductor(object):
|
|
661
661
|
"record_identifier": psm.record_identifier,
|
662
662
|
"config_file": psm.config_path,
|
663
663
|
"output_schema": psm.cfg["_schema_path"],
|
664
|
-
"pephub_path": psm.cfg["pephub_path"],
|
665
664
|
}
|
666
665
|
filtered_namespace = {k: v for k, v in full_namespace.items() if v}
|
667
666
|
return YAMLConfigManager(filtered_namespace)
|
looper/divvy.py
CHANGED
@@ -111,12 +111,9 @@ class ComputingConfiguration(YAMLConfigManager):
|
|
111
111
|
|
112
112
|
:return str: path to folder with default submission templates
|
113
113
|
"""
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
return os.path.join(
|
118
|
-
os.path.dirname(__file__), "default_config", "divvy_templates"
|
119
|
-
)
|
114
|
+
return os.path.join(
|
115
|
+
os.path.dirname(__file__), "default_config", "divvy_templates"
|
116
|
+
)
|
120
117
|
|
121
118
|
def activate_package(self, package_name):
|
122
119
|
"""
|
@@ -158,18 +155,11 @@ class ComputingConfiguration(YAMLConfigManager):
|
|
158
155
|
# but now, it makes more sense to do it here so we can piggyback on
|
159
156
|
# the default update() method and not even have to do that.
|
160
157
|
if not os.path.isabs(self.compute["submission_template"]):
|
161
|
-
|
162
158
|
try:
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
)
|
168
|
-
else:
|
169
|
-
self.compute["submission_template"] = os.path.join(
|
170
|
-
os.path.dirname(self.default_config_file),
|
171
|
-
self.compute["submission_template"],
|
172
|
-
)
|
159
|
+
self.compute["submission_template"] = os.path.join(
|
160
|
+
os.path.dirname(self.default_config_file),
|
161
|
+
self.compute["submission_template"],
|
162
|
+
)
|
173
163
|
except AttributeError as e:
|
174
164
|
# Environment and environment compute should at least have been
|
175
165
|
# set as null-valued attributes, so execution here is an error.
|
@@ -210,11 +200,6 @@ class ComputingConfiguration(YAMLConfigManager):
|
|
210
200
|
"""
|
211
201
|
return self.compute
|
212
202
|
|
213
|
-
@property
|
214
|
-
def compute_packages(self):
|
215
|
-
|
216
|
-
return self["compute_packages"]
|
217
|
-
|
218
203
|
def list_compute_packages(self):
|
219
204
|
"""
|
220
205
|
Returns a list of available compute packages.
|
@@ -411,13 +396,11 @@ def divvy_init(config_path, template_config_path):
|
|
411
396
|
_LOGGER.error("You must specify a template config file path.")
|
412
397
|
return
|
413
398
|
|
414
|
-
if not os.path.isabs(config_path):
|
415
|
-
config_path = os.path.abspath(config_path)
|
416
|
-
|
417
399
|
if config_path and not os.path.exists(config_path):
|
400
|
+
# dcc.write(config_path)
|
418
401
|
# Init should *also* write the templates.
|
419
402
|
dest_folder = os.path.dirname(config_path)
|
420
|
-
copytree(os.path.dirname(template_config_path), dest_folder
|
403
|
+
copytree(os.path.dirname(template_config_path), dest_folder)
|
421
404
|
template_subfolder = os.path.join(dest_folder, "divvy_templates")
|
422
405
|
_LOGGER.info("Wrote divvy templates to folder: {}".format(template_subfolder))
|
423
406
|
new_template = os.path.join(
|
looper/exceptions.py
CHANGED
@@ -15,7 +15,6 @@ _all__ = [
|
|
15
15
|
"PipelineInterfaceConfigError",
|
16
16
|
"PipelineInterfaceRequirementsError",
|
17
17
|
"MisconfigurationException",
|
18
|
-
"LooperReportError",
|
19
18
|
]
|
20
19
|
|
21
20
|
|
@@ -32,7 +31,7 @@ class SampleFailedException(LooperError):
|
|
32
31
|
|
33
32
|
|
34
33
|
class MisconfigurationException(LooperError):
|
35
|
-
"""
|
34
|
+
"""Duplication of pipeline identifier precludes unique pipeline ref."""
|
36
35
|
|
37
36
|
def __init__(self, key):
|
38
37
|
super(MisconfigurationException, self).__init__(key)
|
@@ -110,10 +109,3 @@ class PipelineInterfaceRequirementsError(LooperError):
|
|
110
109
|
)
|
111
110
|
)
|
112
111
|
self.error_specs = typename_by_requirement
|
113
|
-
|
114
|
-
|
115
|
-
class LooperReportError(LooperError):
|
116
|
-
"""Looper reporting errors"""
|
117
|
-
|
118
|
-
def __init__(self, reason):
|
119
|
-
super(LooperReportError, self).__init__(reason)
|
looper/looper.py
CHANGED
@@ -46,7 +46,6 @@ from .utils import (
|
|
46
46
|
sample_folder,
|
47
47
|
)
|
48
48
|
from pipestat.reports import get_file_for_table
|
49
|
-
from pipestat.exceptions import PipestatSummarizeError
|
50
49
|
|
51
50
|
_PKGNAME = "looper"
|
52
51
|
_LOGGER = logging.getLogger(_PKGNAME)
|
@@ -95,19 +94,11 @@ class Checker(Executor):
|
|
95
94
|
|
96
95
|
for piface in self.prj.project_pipeline_interfaces:
|
97
96
|
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
98
|
-
|
99
|
-
|
100
|
-
for pl_name, psm in psms.items():
|
101
|
-
all_project_level_records = psm.select_records()
|
102
|
-
for record in all_project_level_records["records"]:
|
103
|
-
s = piface.psm.get_status(
|
104
|
-
record_identifier=record["record_identifier"]
|
105
|
-
)
|
97
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
98
|
+
s = piface.psm.get_status() or "unknown"
|
106
99
|
status.setdefault(piface.psm.pipeline_name, {})
|
107
|
-
status[piface.psm.pipeline_name][
|
108
|
-
_LOGGER.debug(
|
109
|
-
f"{self.prj.name} ({record['record_identifier']}): {s}"
|
110
|
-
)
|
100
|
+
status[piface.psm.pipeline_name][self.prj.name] = s
|
101
|
+
_LOGGER.debug(f"{self.prj.name} ({piface.psm.pipeline_name}): {s}")
|
111
102
|
|
112
103
|
else:
|
113
104
|
for sample in self.prj.samples:
|
@@ -568,26 +559,15 @@ class Reporter(Executor):
|
|
568
559
|
|
569
560
|
portable = args.portable
|
570
561
|
|
571
|
-
report_dir = getattr(args, "report_dir", None)
|
572
|
-
|
573
562
|
psms = {}
|
574
563
|
|
575
564
|
if project_level:
|
576
565
|
|
577
566
|
for piface in self.prj.project_pipeline_interfaces:
|
578
567
|
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
try:
|
583
|
-
report_directory = psm.summarize(
|
584
|
-
looper_samples=self.prj.samples,
|
585
|
-
portable=portable,
|
586
|
-
output_dir=report_dir,
|
587
|
-
)
|
588
|
-
except PipestatSummarizeError as e:
|
589
|
-
raise LooperReportError(
|
590
|
-
f"Looper report error due to the following exception: {e}"
|
568
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
569
|
+
report_directory = piface.psm.summarize(
|
570
|
+
looper_samples=self.prj.samples, portable=portable
|
591
571
|
)
|
592
572
|
print(f"Report directory: {report_directory}")
|
593
573
|
self.debug["report_directory"] = report_directory
|
@@ -595,21 +575,12 @@ class Reporter(Executor):
|
|
595
575
|
else:
|
596
576
|
for piface in self.prj.pipeline_interfaces:
|
597
577
|
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
try:
|
602
|
-
report_directory = psm.summarize(
|
603
|
-
looper_samples=self.prj.samples,
|
604
|
-
portable=portable,
|
605
|
-
output_dir=report_dir,
|
606
|
-
)
|
607
|
-
except PipestatSummarizeError as e:
|
608
|
-
raise LooperReportError(
|
609
|
-
f"Looper report error due to the following exception: {e}"
|
578
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
579
|
+
report_directory = piface.psm.summarize(
|
580
|
+
looper_samples=self.prj.samples, portable=portable
|
610
581
|
)
|
611
|
-
|
612
|
-
|
582
|
+
print(f"Report directory: {report_directory}")
|
583
|
+
self.debug["report_directory"] = report_directory
|
613
584
|
return self.debug
|
614
585
|
|
615
586
|
|
@@ -647,23 +618,18 @@ class Tabulator(Executor):
|
|
647
618
|
def __call__(self, args):
|
648
619
|
# p = self.prj
|
649
620
|
project_level = getattr(args, "project", None)
|
650
|
-
report_dir = getattr(args, "report_dir", None)
|
651
621
|
results = []
|
652
622
|
psms = {}
|
653
623
|
if project_level:
|
654
624
|
for piface in self.prj.project_pipeline_interfaces:
|
655
625
|
if piface.psm.pipeline_type == PipelineLevel.PROJECT.value:
|
656
|
-
|
657
|
-
|
658
|
-
for pl_name, psm in psms.items():
|
659
|
-
results = psm.table(output_dir=report_dir)
|
626
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
627
|
+
results = piface.psm.table()
|
660
628
|
else:
|
661
629
|
for piface in self.prj.pipeline_interfaces:
|
662
630
|
if piface.psm.pipeline_type == PipelineLevel.SAMPLE.value:
|
663
|
-
|
664
|
-
|
665
|
-
for pl_name, psm in psms.items():
|
666
|
-
results = psm.table(output_dir=report_dir)
|
631
|
+
psms[piface.psm.pipeline_name] = piface.psm
|
632
|
+
results = piface.psm.table()
|
667
633
|
# Results contains paths to stats and object summaries.
|
668
634
|
return results
|
669
635
|
|
looper/utils.py
CHANGED
@@ -17,7 +17,6 @@ from ubiquerg import convert_value, expandpath, parse_registry_path, deep_update
|
|
17
17
|
from pephubclient.constants import RegistryPath
|
18
18
|
from pydantic import ValidationError
|
19
19
|
from yacman import load_yaml
|
20
|
-
from yaml.parser import ParserError
|
21
20
|
|
22
21
|
from .const import *
|
23
22
|
from .command_models.commands import SUPPORTED_COMMANDS
|
@@ -264,29 +263,31 @@ def enrich_args_via_cfg(
|
|
264
263
|
cli_modifiers=None,
|
265
264
|
):
|
266
265
|
"""
|
267
|
-
Read in a looper dotfile
|
266
|
+
Read in a looper dotfile and set arguments.
|
268
267
|
|
269
|
-
Priority order: CLI > dotfile/config >
|
268
|
+
Priority order: CLI > dotfile/config > parser default
|
270
269
|
|
271
270
|
:param subcommand name: the name of the command used
|
272
271
|
:param argparse.Namespace parser_args: parsed args by the original parser
|
273
|
-
:param argparse.Namespace aux_parser: parsed args by the
|
272
|
+
:param argparse.Namespace aux_parser: parsed args by the a parser
|
274
273
|
with defaults suppressed
|
275
|
-
:param dict test_args: dict of args used for pytesting
|
276
|
-
:param dict cli_modifiers: dict of args existing if user supplied cli args in looper config file
|
277
274
|
:return argparse.Namespace: selected argument values
|
278
275
|
"""
|
279
|
-
|
280
|
-
# Did the user provide arguments in the PEP config?
|
281
276
|
cfg_args_all = (
|
282
277
|
_get_subcommand_args(subcommand_name, parser_args)
|
283
278
|
if os.path.exists(parser_args.pep_config)
|
284
279
|
else dict()
|
285
280
|
)
|
286
|
-
|
281
|
+
|
282
|
+
# If user provided project-level modifiers in the looper config, they are prioritized
|
283
|
+
if cfg_args_all:
|
284
|
+
for key, value in cfg_args_all.items():
|
285
|
+
if getattr(parser_args, key, None):
|
286
|
+
new_value = getattr(parser_args, key)
|
287
|
+
cfg_args_all[key] = new_value
|
288
|
+
else:
|
287
289
|
cfg_args_all = {}
|
288
290
|
|
289
|
-
# Did the user provide arguments/modifiers in the looper config file?
|
290
291
|
looper_config_cli_modifiers = None
|
291
292
|
if cli_modifiers:
|
292
293
|
if str(subcommand_name) in cli_modifiers:
|
@@ -311,13 +312,6 @@ def enrich_args_via_cfg(
|
|
311
312
|
else:
|
312
313
|
cli_args, _ = aux_parser.parse_known_args()
|
313
314
|
|
314
|
-
# If any CLI args were provided, make sure they take priority
|
315
|
-
if cli_args:
|
316
|
-
r = getattr(cli_args, subcommand_name)
|
317
|
-
for k, v in cfg_args_all.items():
|
318
|
-
if k in r:
|
319
|
-
cfg_args_all[k] = getattr(r, k)
|
320
|
-
|
321
315
|
def set_single_arg(argname, default_source_namespace, result_namespace):
|
322
316
|
if argname not in POSITIONAL or not hasattr(result, argname):
|
323
317
|
if argname in cli_args:
|
@@ -330,8 +324,6 @@ def enrich_args_via_cfg(
|
|
330
324
|
elif cfg_args_all is not None and argname in cfg_args_all:
|
331
325
|
if isinstance(cfg_args_all[argname], list):
|
332
326
|
r = [convert_value(i) for i in cfg_args_all[argname]]
|
333
|
-
elif isinstance(cfg_args_all[argname], dict):
|
334
|
-
r = cfg_args_all[argname]
|
335
327
|
else:
|
336
328
|
r = convert_value(cfg_args_all[argname])
|
337
329
|
else:
|
@@ -440,8 +432,9 @@ def init_generic_pipeline(pipelinepath: Optional[str] = None):
|
|
440
432
|
generic_pipeline_dict = {
|
441
433
|
"pipeline_name": "default_pipeline_name",
|
442
434
|
"output_schema": "output_schema.yaml",
|
435
|
+
"var_templates": {"pipeline": "{looper.piface_dir}/count_lines.sh"},
|
443
436
|
"sample_interface": {
|
444
|
-
"command_template": "{
|
437
|
+
"command_template": "{pipeline.var_templates.pipeline} {sample.file} "
|
445
438
|
"--output-parent {looper.sample_output_folder}"
|
446
439
|
},
|
447
440
|
}
|
@@ -634,6 +627,8 @@ def looper_config_tutorial():
|
|
634
627
|
console.print("Use [yellow]`looper run`[/yellow] afterwards to run the pipeline.")
|
635
628
|
console.print("Press [yellow]^C[/yellow] at any time to quit.\n")
|
636
629
|
|
630
|
+
console.input("> ... ")
|
631
|
+
|
637
632
|
DEFAULTS = { # What you get if you just press enter
|
638
633
|
"pep_config": "databio/example",
|
639
634
|
"output_dir": "results",
|
@@ -641,58 +636,73 @@ def looper_config_tutorial():
|
|
641
636
|
"project_name": os.path.basename(os.getcwd()),
|
642
637
|
}
|
643
638
|
|
644
|
-
|
645
|
-
console.input(f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >")
|
646
|
-
or DEFAULTS["project_name"]
|
647
|
-
)
|
639
|
+
creating = True
|
648
640
|
|
649
|
-
|
650
|
-
|
651
|
-
|
641
|
+
while creating:
|
642
|
+
cfg["project_name"] = (
|
643
|
+
console.input(
|
644
|
+
f"Project name: [yellow]({DEFAULTS['project_name']})[/yellow] >"
|
645
|
+
)
|
646
|
+
or DEFAULTS["project_name"]
|
652
647
|
)
|
653
|
-
or DEFAULTS["pep_config"]
|
654
|
-
)
|
655
648
|
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
|
649
|
+
cfg["pep_config"] = (
|
650
|
+
console.input(
|
651
|
+
f"Registry path or file path to PEP: [yellow]({DEFAULTS['pep_config']})[/yellow] >"
|
652
|
+
)
|
653
|
+
or DEFAULTS["pep_config"]
|
661
654
|
)
|
662
655
|
|
663
|
-
|
664
|
-
|
665
|
-
|
666
|
-
|
667
|
-
or DEFAULTS["output_dir"]
|
668
|
-
)
|
656
|
+
if not os.path.exists(cfg["pep_config"]):
|
657
|
+
console.print(
|
658
|
+
f"Warning: PEP file does not exist at [yellow]'{cfg['pep_config']}[/yellow]'"
|
659
|
+
)
|
669
660
|
|
670
|
-
|
671
|
-
piface_paths = []
|
672
|
-
while add_more_pifaces:
|
673
|
-
piface_path = (
|
661
|
+
cfg["output_dir"] = (
|
674
662
|
console.input(
|
675
|
-
"
|
663
|
+
f"Path to output directory: [yellow]({DEFAULTS['output_dir']})[/yellow] >"
|
676
664
|
)
|
677
|
-
or
|
665
|
+
or DEFAULTS["output_dir"]
|
678
666
|
)
|
679
|
-
if piface_path is None:
|
680
|
-
if piface_paths == []:
|
681
|
-
piface_paths.append(DEFAULTS["piface_path"])
|
682
|
-
add_more_pifaces = False
|
683
|
-
else:
|
684
|
-
piface_paths.append(piface_path)
|
685
667
|
|
686
|
-
|
668
|
+
add_more_pifaces = True
|
669
|
+
piface_paths = []
|
670
|
+
while add_more_pifaces:
|
671
|
+
piface_path = (
|
672
|
+
console.input(
|
673
|
+
"Add each path to a pipeline interface: [yellow](pipeline_interface.yaml)[/yellow] >"
|
674
|
+
)
|
675
|
+
or None
|
676
|
+
)
|
677
|
+
if piface_path is None:
|
678
|
+
if piface_paths == []:
|
679
|
+
piface_paths.append(DEFAULTS["piface_path"])
|
680
|
+
add_more_pifaces = False
|
681
|
+
else:
|
682
|
+
piface_paths.append(piface_path)
|
683
|
+
|
684
|
+
console.print("\n")
|
687
685
|
|
688
|
-
|
689
|
-
|
690
|
-
[yellow]pep_config:[/yellow] {cfg['pep_config']}
|
691
|
-
[yellow]output_dir:[/yellow] {cfg['output_dir']}
|
692
|
-
[yellow]pipeline_interfaces:[/yellow]
|
693
|
-
|
694
|
-
"""
|
695
|
-
|
686
|
+
console.print(
|
687
|
+
f"""\
|
688
|
+
[yellow]pep_config:[/yellow] {cfg['pep_config']}
|
689
|
+
[yellow]output_dir:[/yellow] {cfg['output_dir']}
|
690
|
+
[yellow]pipeline_interfaces:[/yellow]
|
691
|
+
- {piface_paths}
|
692
|
+
"""
|
693
|
+
)
|
694
|
+
|
695
|
+
console.print(
|
696
|
+
"[bold]Does this look good?[/bold] [bold green]Y[/bold green]/[red]n[/red]..."
|
697
|
+
)
|
698
|
+
selection = None
|
699
|
+
while selection not in ["y", "n"]:
|
700
|
+
selection = console.input("\nSelection: ").lower().strip()
|
701
|
+
if selection == "n":
|
702
|
+
console.print("Starting over...")
|
703
|
+
pass
|
704
|
+
if selection == "y":
|
705
|
+
creating = False
|
696
706
|
|
697
707
|
for piface_path in piface_paths:
|
698
708
|
if not os.path.exists(piface_path):
|
@@ -717,7 +727,7 @@ def looper_config_tutorial():
|
|
717
727
|
looper_config_dict = {}
|
718
728
|
looper_config_dict["pep_config"] = cfg["pep_config"]
|
719
729
|
looper_config_dict["output_dir"] = cfg["output_dir"]
|
720
|
-
looper_config_dict["pipeline_interfaces"] = piface_paths
|
730
|
+
looper_config_dict["pipeline_interfaces"] = [piface_paths]
|
721
731
|
|
722
732
|
with open(looper_cfg_path, "w") as fp:
|
723
733
|
yaml.dump(looper_config_dict, fp)
|
@@ -737,15 +747,7 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
|
|
737
747
|
|
738
748
|
if piface_path is None:
|
739
749
|
return None, None
|
740
|
-
|
741
|
-
piface_path = expandpath(piface_path)
|
742
|
-
except TypeError as e:
|
743
|
-
_LOGGER.warning(
|
744
|
-
f"Pipeline interface not found at given path: {piface_path}. Type Error: "
|
745
|
-
+ str(e)
|
746
|
-
)
|
747
|
-
return None, None
|
748
|
-
|
750
|
+
piface_path = expandpath(piface_path)
|
749
751
|
if not os.path.isabs(piface_path):
|
750
752
|
piface_path = os.path.realpath(
|
751
753
|
os.path.join(os.path.dirname(looper_config_path), piface_path)
|
@@ -753,7 +755,6 @@ def determine_pipeline_type(piface_path: str, looper_config_path: str):
|
|
753
755
|
try:
|
754
756
|
piface_dict = load_yaml(piface_path)
|
755
757
|
except FileNotFoundError:
|
756
|
-
_LOGGER.warning(f"Pipeline interface not found at given path: {piface_path}")
|
757
758
|
return None, None
|
758
759
|
|
759
760
|
pipeline_types = []
|
@@ -782,18 +783,12 @@ def read_looper_config_file(looper_config_path: str) -> dict:
|
|
782
783
|
:raise MisconfigurationException: incorrect configuration.
|
783
784
|
"""
|
784
785
|
return_dict = {}
|
785
|
-
|
786
|
-
|
787
|
-
with open(looper_config_path, "r") as dotfile:
|
788
|
-
dp_data = yaml.safe_load(dotfile)
|
789
|
-
except ParserError as e:
|
790
|
-
_LOGGER.warning(
|
791
|
-
"Could not load looper config file due to the following exception"
|
792
|
-
)
|
793
|
-
raise ParserError(context=str(e))
|
786
|
+
with open(looper_config_path, "r") as dotfile:
|
787
|
+
dp_data = yaml.safe_load(dotfile)
|
794
788
|
|
795
789
|
if PEP_CONFIG_KEY in dp_data:
|
796
790
|
return_dict[PEP_CONFIG_KEY] = dp_data[PEP_CONFIG_KEY]
|
791
|
+
|
797
792
|
else:
|
798
793
|
raise MisconfigurationException(
|
799
794
|
f"Looper dotfile ({looper_config_path}) is missing '{PEP_CONFIG_KEY}' key"
|
@@ -851,7 +846,6 @@ def read_looper_config_file(looper_config_path: str) -> dict:
|
|
851
846
|
for k, v in return_dict.items():
|
852
847
|
if k == SAMPLE_PL_ARG or k == PROJECT_PL_ARG:
|
853
848
|
# Pipeline interfaces are resolved at a later point. Do it there only to maintain consistency. #474
|
854
|
-
|
855
849
|
pass
|
856
850
|
if isinstance(v, str):
|
857
851
|
v = expandpath(v)
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.1
|
2
2
|
Name: looper
|
3
|
-
Version: 2.0.
|
3
|
+
Version: 2.0.0a1
|
4
4
|
Summary: A pipeline submission engine that parses sample inputs and submits pipelines for each sample.
|
5
5
|
Home-page: https://github.com/pepkit/looper
|
6
6
|
Author: Nathan Sheffield, Vince Reuter, Michal Stolarczyk, Johanna Klughammer, Andre Rendeiro
|
@@ -15,29 +15,21 @@ Classifier: Programming Language :: Python :: 3.11
|
|
15
15
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
16
16
|
Description-Content-Type: text/markdown
|
17
17
|
License-File: LICENSE.txt
|
18
|
-
Requires-Dist: colorama>=0.3.9
|
19
|
-
Requires-Dist:
|
18
|
+
Requires-Dist: colorama >=0.3.9
|
19
|
+
Requires-Dist: divvy >=0.5.0
|
20
|
+
Requires-Dist: eido >=0.2.1
|
20
21
|
Requires-Dist: jinja2
|
21
|
-
Requires-Dist: logmuse>=0.2.0
|
22
|
-
Requires-Dist: pandas>=2.0.2
|
23
|
-
Requires-Dist: pephubclient>=0.4.0
|
24
|
-
Requires-Dist: pipestat>=0.
|
25
|
-
Requires-Dist: peppy>=0.40.
|
26
|
-
Requires-Dist: pyyaml>=3.12
|
27
|
-
Requires-Dist: rich>=9.10.0
|
28
|
-
Requires-Dist: ubiquerg>=0.8.1a1
|
29
|
-
Requires-Dist: yacman==0.9.3
|
30
|
-
Requires-Dist: pydantic-argparse>=0.9.0
|
22
|
+
Requires-Dist: logmuse >=0.2.0
|
23
|
+
Requires-Dist: pandas >=2.0.2
|
24
|
+
Requires-Dist: pephubclient >=0.4.0
|
25
|
+
Requires-Dist: pipestat >=0.9.2
|
26
|
+
Requires-Dist: peppy >=0.40.2
|
27
|
+
Requires-Dist: pyyaml >=3.12
|
28
|
+
Requires-Dist: rich >=9.10.0
|
29
|
+
Requires-Dist: ubiquerg >=0.8.1a1
|
30
|
+
Requires-Dist: yacman ==0.9.3
|
31
|
+
Requires-Dist: pydantic-argparse >=0.9.0
|
31
32
|
Requires-Dist: psutil
|
32
|
-
Dynamic: author
|
33
|
-
Dynamic: classifier
|
34
|
-
Dynamic: description
|
35
|
-
Dynamic: description-content-type
|
36
|
-
Dynamic: home-page
|
37
|
-
Dynamic: keywords
|
38
|
-
Dynamic: license
|
39
|
-
Dynamic: requires-dist
|
40
|
-
Dynamic: summary
|
41
33
|
|
42
34
|
# <img src="docs/img/looper_logo.svg" alt="looper logo" height="70">
|
43
35
|
|
@@ -1,24 +1,24 @@
|
|
1
1
|
looper/__init__.py,sha256=f_z9YY4ibOk7eyWoaViH_VaCXMlPQeiftbnibSFj-3E,1333
|
2
2
|
looper/__main__.py,sha256=OOCmI-dPUvInnJHkHNMf54cblNJ3Yl9ELOwZcfOXmD8,240
|
3
|
-
looper/_version.py,sha256=
|
4
|
-
looper/cli_divvy.py,sha256=
|
5
|
-
looper/cli_pydantic.py,sha256=
|
6
|
-
looper/conductor.py,sha256=
|
3
|
+
looper/_version.py,sha256=wJ2KiFRb6QXaKu1tfxwKmo9CE2xP4f3BkEUYoIJeUkQ,121
|
4
|
+
looper/cli_divvy.py,sha256=J07x83sqC4jJeu3_yS6KOARPWmwKGAV7JvN33T5zDac,5907
|
5
|
+
looper/cli_pydantic.py,sha256=saAD0dwizLi_VajVKElQmYRqtn0Yyl_r712znxZMxRQ,14209
|
6
|
+
looper/conductor.py,sha256=oNfqANA2tIhQTAQFxd-rQ4ccPA60D933EGo3rfbJGFo,35061
|
7
7
|
looper/const.py,sha256=OscEELQsyLKlSrmwuXfyLRwpAUJUEpGD2UxBeLJDXgw,8703
|
8
|
-
looper/divvy.py,sha256=
|
9
|
-
looper/exceptions.py,sha256=
|
10
|
-
looper/looper.py,sha256=
|
8
|
+
looper/divvy.py,sha256=5x8hV1lT5tEQdAUtVjn0rNwYnJroNij0RyDn-wHf4QE,15251
|
9
|
+
looper/exceptions.py,sha256=r6SKKt-m8CXQnXGDnuiwoA6zBJhIZflygBKjX4RCloI,3419
|
10
|
+
looper/looper.py,sha256=ZWTulMz6NobnYFUjev513TJwXqknrb4_gZrV-a_fT9g,30041
|
11
11
|
looper/parser_types.py,sha256=d3FHt54f9jo9VZMr5SQkbghcAdABqiYZW2JBGO5EBnw,2327
|
12
12
|
looper/pipeline_interface.py,sha256=mN4-XICyZzuVLTOq3b0ijppYe6ib_Ljlyf6KxZCJh2A,14537
|
13
13
|
looper/plugins.py,sha256=MaMdPmK9U_4FkNJE5kccohBbY1i2qj1NTEucubFOJek,5747
|
14
14
|
looper/processed_project.py,sha256=jZxoMYafvr-OHFxylc5ivGty1VwXBZhl0kgoFkY-174,9837
|
15
15
|
looper/project.py,sha256=SFHdi58eRBWtye5lUFhwzBcG7ejrMurmDzmkrC3XAic,34339
|
16
|
-
looper/utils.py,sha256
|
16
|
+
looper/utils.py,sha256=VXKaEYXW0XEQNy__hNBv42i-sz5qB7QiwpXzhXs_3Wk,39556
|
17
17
|
looper/command_models/DEVELOPER.md,sha256=eRxnrO-vqNJjExzamXKEq5wr_-Zw6PQEwkS9RPinYrk,2775
|
18
18
|
looper/command_models/README.md,sha256=3RGegeZlTZYnhcHXRu6bdI_81WZom2q7QYMV-KGYY7U,588
|
19
19
|
looper/command_models/__init__.py,sha256=6QWC2TewowEL7dATli5YpMmFWuXaLEPktofJCXkYUBI,187
|
20
|
-
looper/command_models/arguments.py,sha256=
|
21
|
-
looper/command_models/commands.py,sha256=
|
20
|
+
looper/command_models/arguments.py,sha256=sRrJWCSQmnjGLnOo-Wl6_PnvTZz8VIWII79svI3BqFk,8653
|
21
|
+
looper/command_models/commands.py,sha256=EvKyjNdUBspXnOUMprxIY0C4VPka2PBj8CJgtd5Ya9w,9680
|
22
22
|
looper/default_config/divvy_config.yaml,sha256=wK5kLDGBV2wwoyqg2rl3X8SXjds4x0mwBUjUzF1Ln7g,1705
|
23
23
|
looper/default_config/divvy_templates/localhost_bulker_template.sub,sha256=yn5VB9Brt7Hck9LT17hD2o8Kn-76gYJQk_A-8C1Gr4k,164
|
24
24
|
looper/default_config/divvy_templates/localhost_docker_template.sub,sha256=XRr7AlR7-TP1L3hyBMfka_RgWRL9vzOlS5Kd1xSNwT0,183
|
@@ -60,9 +60,9 @@ looper/schemas/divvy_config_schema.yaml,sha256=7GJfKLc3VX4RGjHnOE1zxwsHXhj_ur9za
|
|
60
60
|
looper/schemas/pipeline_interface_schema_generic.yaml,sha256=3YfKFyRUIwxG41FEidR1dXe9IU6ye51LSUBfSpmMuss,1773
|
61
61
|
looper/schemas/pipeline_interface_schema_project.yaml,sha256=-ZWyA0lKXWik3obuLNVk3IsAZYfbLVbCDvJnD-Fcluo,1567
|
62
62
|
looper/schemas/pipeline_interface_schema_sample.yaml,sha256=x0OwVnijJpvm50DscvvJujdK4UAI7d71pqVemQS-D-0,1564
|
63
|
-
looper-2.0.
|
64
|
-
looper-2.0.
|
65
|
-
looper-2.0.
|
66
|
-
looper-2.0.
|
67
|
-
looper-2.0.
|
68
|
-
looper-2.0.
|
63
|
+
looper-2.0.0a1.dist-info/LICENSE.txt,sha256=oB6ZGDa4kcznznJKJsLLFFcOZyi8Y6e2Jv0rJozgp-I,1269
|
64
|
+
looper-2.0.0a1.dist-info/METADATA,sha256=OZPEqy9PPCjzJ110u-HazaRkaUIiuesCZMC3jpfl2vQ,1800
|
65
|
+
looper-2.0.0a1.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
|
66
|
+
looper-2.0.0a1.dist-info/entry_points.txt,sha256=ejZpghZG3OoTK69u9rTW-yLyI6SC63bBTUb-Vw26HG4,87
|
67
|
+
looper-2.0.0a1.dist-info/top_level.txt,sha256=I0Yf7djsoQAMzwHBbDiQi9hGtq4Z41_Ma5CX8qXG8Y8,7
|
68
|
+
looper-2.0.0a1.dist-info/RECORD,,
|
File without changes
|
File without changes
|