looper 1.5.0__py3-none-any.whl → 1.6.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- looper/__init__.py +3 -498
- looper/__main__.py +2 -2
- looper/_version.py +1 -1
- looper/cli_divvy.py +182 -0
- looper/cli_looper.py +776 -0
- looper/conductor.py +53 -206
- looper/const.py +51 -3
- looper/divvy.py +28 -196
- looper/exceptions.py +18 -0
- looper/looper.py +177 -612
- looper/plugins.py +160 -0
- looper/processed_project.py +1 -1
- looper/project.py +229 -117
- looper/utils.py +119 -43
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/METADATA +6 -6
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/RECORD +20 -20
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/WHEEL +1 -1
- looper/html_reports.py +0 -1057
- looper/html_reports_pipestat.py +0 -924
- looper/html_reports_project_pipestat.py +0 -269
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/LICENSE.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/entry_points.txt +0 -0
- {looper-1.5.0.dist-info → looper-1.6.0a1.dist-info}/top_level.txt +0 -0
looper/cli_divvy.py
ADDED
@@ -0,0 +1,182 @@
|
|
1
|
+
import logmuse
|
2
|
+
import os
|
3
|
+
import sys
|
4
|
+
import yaml
|
5
|
+
from yaml import SafeLoader
|
6
|
+
from ubiquerg import is_writable, VersionInHelpParser
|
7
|
+
from .const import (
|
8
|
+
DEFAULT_COMPUTE_RESOURCES_NAME,
|
9
|
+
DEFAULT_CONFIG_FILEPATH,
|
10
|
+
)
|
11
|
+
from .divvy import select_divvy_config, ComputingConfiguration, divvy_init
|
12
|
+
|
13
|
+
|
14
|
+
def build_argparser():
|
15
|
+
"""
|
16
|
+
Builds argument parser.
|
17
|
+
|
18
|
+
:return argparse.ArgumentParser
|
19
|
+
"""
|
20
|
+
|
21
|
+
banner = (
|
22
|
+
"%(prog)s - write compute job scripts that can be submitted to "
|
23
|
+
"any computing resource"
|
24
|
+
)
|
25
|
+
additional_description = "\nhttps://divvy.databio.org"
|
26
|
+
|
27
|
+
parser = VersionInHelpParser(
|
28
|
+
prog="divvy",
|
29
|
+
description=banner,
|
30
|
+
epilog=additional_description,
|
31
|
+
# version=__version__,
|
32
|
+
)
|
33
|
+
|
34
|
+
subparsers = parser.add_subparsers(dest="command")
|
35
|
+
|
36
|
+
def add_subparser(cmd, description):
|
37
|
+
return subparsers.add_parser(cmd, description=description, help=description)
|
38
|
+
|
39
|
+
subparser_messages = {
|
40
|
+
"init": "Initialize a new divvy config file",
|
41
|
+
"list": "List available compute packages",
|
42
|
+
"write": "Write a job script",
|
43
|
+
"submit": "Write and then submit a job script",
|
44
|
+
"inspect": "Inspect compute package",
|
45
|
+
}
|
46
|
+
|
47
|
+
sps = {}
|
48
|
+
for cmd, desc in subparser_messages.items():
|
49
|
+
sps[cmd] = add_subparser(cmd, desc)
|
50
|
+
# sps[cmd].add_argument(
|
51
|
+
# "config", nargs="?", default=None,
|
52
|
+
# help="Divvy configuration file.")
|
53
|
+
|
54
|
+
for sp in [sps["list"], sps["write"], sps["submit"], sps["inspect"]]:
|
55
|
+
sp.add_argument(
|
56
|
+
"config", nargs="?", default=None, help="Divvy configuration file."
|
57
|
+
)
|
58
|
+
|
59
|
+
sps["init"].add_argument("config", default=None, help="Divvy configuration file.")
|
60
|
+
|
61
|
+
for sp in [sps["inspect"]]:
|
62
|
+
sp.add_argument(
|
63
|
+
"-p",
|
64
|
+
"--package",
|
65
|
+
default=DEFAULT_COMPUTE_RESOURCES_NAME,
|
66
|
+
help="Select from available compute packages",
|
67
|
+
)
|
68
|
+
|
69
|
+
for sp in [sps["write"], sps["submit"]]:
|
70
|
+
sp.add_argument(
|
71
|
+
"-s",
|
72
|
+
"--settings",
|
73
|
+
help="YAML file with job settings to populate the template",
|
74
|
+
)
|
75
|
+
|
76
|
+
sp.add_argument(
|
77
|
+
"-p",
|
78
|
+
"--package",
|
79
|
+
default=DEFAULT_COMPUTE_RESOURCES_NAME,
|
80
|
+
help="Select from available compute packages",
|
81
|
+
)
|
82
|
+
|
83
|
+
sp.add_argument(
|
84
|
+
"-c",
|
85
|
+
"--compute",
|
86
|
+
nargs="+",
|
87
|
+
default=None,
|
88
|
+
help="Extra key=value variable pairs",
|
89
|
+
)
|
90
|
+
|
91
|
+
# sp.add_argument(
|
92
|
+
# "-t", "--template",
|
93
|
+
# help="Provide a template file (not yet implemented).")
|
94
|
+
|
95
|
+
sp.add_argument(
|
96
|
+
"-o", "--outfile", required=False, default=None, help="Output filepath"
|
97
|
+
)
|
98
|
+
|
99
|
+
return parser
|
100
|
+
|
101
|
+
|
102
|
+
def main():
|
103
|
+
"""Primary workflow for divvy CLI"""
|
104
|
+
|
105
|
+
parser = logmuse.add_logging_options(build_argparser())
|
106
|
+
# args, remaining_args = parser.parse_known_args()
|
107
|
+
args = parser.parse_args()
|
108
|
+
|
109
|
+
logger_kwargs = {"level": args.verbosity, "devmode": args.logdev}
|
110
|
+
logmuse.init_logger("yacman", **logger_kwargs)
|
111
|
+
global _LOGGER
|
112
|
+
_LOGGER = logmuse.logger_via_cli(args)
|
113
|
+
|
114
|
+
if not args.command:
|
115
|
+
parser.print_help()
|
116
|
+
_LOGGER.error("No command given")
|
117
|
+
sys.exit(1)
|
118
|
+
|
119
|
+
if args.command == "init":
|
120
|
+
divcfg = args.config
|
121
|
+
_LOGGER.debug("Initializing divvy configuration")
|
122
|
+
is_writable(os.path.dirname(divcfg), check_exist=False)
|
123
|
+
divvy_init(divcfg, DEFAULT_CONFIG_FILEPATH)
|
124
|
+
sys.exit(0)
|
125
|
+
|
126
|
+
_LOGGER.debug("Divvy config: {}".format(args.config))
|
127
|
+
divcfg = select_divvy_config(args.config)
|
128
|
+
_LOGGER.info("Using divvy config: {}".format(divcfg))
|
129
|
+
dcc = ComputingConfiguration(filepath=divcfg)
|
130
|
+
|
131
|
+
if args.command == "list":
|
132
|
+
# Output header via logger and content via print so the user can
|
133
|
+
# redirect the list from stdout if desired without the header as clutter
|
134
|
+
_LOGGER.info("Available compute packages:\n")
|
135
|
+
print("{}".format("\n".join(dcc.list_compute_packages())))
|
136
|
+
sys.exit(1)
|
137
|
+
|
138
|
+
if args.command == "inspect":
|
139
|
+
# Output contents of selected compute package
|
140
|
+
_LOGGER.info("Your compute package template for: " + args.package + "\n")
|
141
|
+
found = False
|
142
|
+
for pkg_name, pkg in dcc.compute_packages.items():
|
143
|
+
if pkg_name == args.package:
|
144
|
+
found = True
|
145
|
+
with open(pkg.submission_template, "r") as f:
|
146
|
+
print(f.read())
|
147
|
+
_LOGGER.info("Submission command is: " + pkg.submission_command + "\n")
|
148
|
+
if pkg_name == "docker":
|
149
|
+
print("Docker args are: " + pkg.docker_args)
|
150
|
+
|
151
|
+
if not found:
|
152
|
+
_LOGGER.info("Package not found. Use 'divvy list' to see list of packages.")
|
153
|
+
sys.exit(1)
|
154
|
+
|
155
|
+
# Any non-divvy arguments will be passed along as key-value pairs
|
156
|
+
# that can be used to populate the template.
|
157
|
+
# keys = [str.replace(x, "--", "") for x in remaining_args[::2]]
|
158
|
+
# cli_vars = dict(zip(keys, remaining_args[1::2]))
|
159
|
+
if args.compute:
|
160
|
+
cli_vars = {y[0]: y[1] for y in [x.split("=") for x in args.compute]}
|
161
|
+
else:
|
162
|
+
cli_vars = {}
|
163
|
+
|
164
|
+
if args.command == "write" or args.command == "submit":
|
165
|
+
try:
|
166
|
+
dcc.activate_package(args.package)
|
167
|
+
except AttributeError:
|
168
|
+
parser.print_help(sys.stderr)
|
169
|
+
sys.exit(1)
|
170
|
+
|
171
|
+
if args.settings:
|
172
|
+
_LOGGER.info("Loading settings file: %s", args.settings)
|
173
|
+
with open(args.settings, "r") as f:
|
174
|
+
vars_groups = [cli_vars, yaml.load(f, SafeLoader)]
|
175
|
+
else:
|
176
|
+
vars_groups = [cli_vars]
|
177
|
+
|
178
|
+
_LOGGER.debug(vars_groups)
|
179
|
+
if args.command == "write":
|
180
|
+
dcc.write_script(args.outfile, vars_groups)
|
181
|
+
elif args.command == "submit":
|
182
|
+
dcc.submit(args.outfile, vars_groups)
|