runem 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runem/command_line.py +7 -6
- runem/config.py +2 -1
- runem/config_parse.py +6 -5
- runem/{job_runner.py → job_execute.py} +17 -10
- runem/job_filter.py +18 -11
- runem/{job_function_python.py → job_wrapper_python.py} +22 -7
- runem/log.py +11 -0
- runem/report.py +39 -14
- runem/run_command.py +8 -6
- runem/runem.py +54 -17
- runem/types.py +3 -2
- {runem-0.0.16.dist-info → runem-0.0.18.dist-info}/METADATA +3 -2
- runem-0.0.18.dist-info/RECORD +25 -0
- runem-0.0.16.dist-info/RECORD +0 -24
- {runem-0.0.16.dist-info → runem-0.0.18.dist-info}/LICENSE +0 -0
- {runem-0.0.16.dist-info → runem-0.0.18.dist-info}/WHEEL +0 -0
- {runem-0.0.16.dist-info → runem-0.0.18.dist-info}/entry_points.txt +0 -0
- {runem-0.0.16.dist-info → runem-0.0.18.dist-info}/top_level.txt +0 -0
runem/command_line.py
CHANGED
@@ -5,6 +5,7 @@ import sys
|
|
5
5
|
import typing
|
6
6
|
|
7
7
|
from runem.config_metadata import ConfigMetadata
|
8
|
+
from runem.log import log
|
8
9
|
from runem.types import JobNames, OptionConfig, Options
|
9
10
|
from runem.utils import printable_set
|
10
11
|
|
@@ -146,7 +147,7 @@ def parse_args(
|
|
146
147
|
|
147
148
|
args = parser.parse_args(argv[1:])
|
148
149
|
|
149
|
-
options: Options =
|
150
|
+
options: Options = initialise_options(config_metadata, args)
|
150
151
|
|
151
152
|
if not _validate_filters(config_metadata, args):
|
152
153
|
sys.exit(1)
|
@@ -176,7 +177,7 @@ def _validate_filters(
|
|
176
177
|
for name, name_list in (("--jobs", args.jobs), ("--not-jobs", args.jobs_excluded)):
|
177
178
|
for job_name in name_list:
|
178
179
|
if job_name not in config_metadata.all_job_names:
|
179
|
-
|
180
|
+
log(
|
180
181
|
(
|
181
182
|
f"ERROR: invalid job-name '{job_name}' for {name}, "
|
182
183
|
f"choose from one of {printable_set(config_metadata.all_job_names)}"
|
@@ -188,7 +189,7 @@ def _validate_filters(
|
|
188
189
|
for name, tag_list in (("--tags", args.tags), ("--not-tags", args.tags_excluded)):
|
189
190
|
for tag in tag_list:
|
190
191
|
if tag not in config_metadata.all_job_tags:
|
191
|
-
|
192
|
+
log(
|
192
193
|
(
|
193
194
|
f"ERROR: invalid tag '{tag}' for {name}, "
|
194
195
|
f"choose from one of {printable_set(config_metadata.all_job_tags)}"
|
@@ -203,7 +204,7 @@ def _validate_filters(
|
|
203
204
|
):
|
204
205
|
for phase in phase_list:
|
205
206
|
if phase not in config_metadata.all_job_phases:
|
206
|
-
|
207
|
+
log(
|
207
208
|
(
|
208
209
|
f"ERROR: invalid phase '{phase}' for {name}, "
|
209
210
|
f"choose from one of {printable_set(config_metadata.all_job_phases)}"
|
@@ -213,7 +214,7 @@ def _validate_filters(
|
|
213
214
|
return True
|
214
215
|
|
215
216
|
|
216
|
-
def
|
217
|
+
def initialise_options(
|
217
218
|
config_metadata: ConfigMetadata,
|
218
219
|
args: argparse.Namespace,
|
219
220
|
) -> Options:
|
@@ -226,7 +227,7 @@ def _initialise_options(
|
|
226
227
|
option["name"]: option["default"] for option in config_metadata.options_config
|
227
228
|
}
|
228
229
|
if config_metadata.options_config and args.overrides_on: # pragma: no branch
|
229
|
-
for option_name in args.overrides_on:
|
230
|
+
for option_name in args.overrides_on: # pragma: no branch
|
230
231
|
options[option_name] = True
|
231
232
|
if config_metadata.options_config and args.overrides_off: # pragma: no branch
|
232
233
|
for option_name in args.overrides_off:
|
runem/config.py
CHANGED
@@ -4,6 +4,7 @@ import typing
|
|
4
4
|
|
5
5
|
import yaml
|
6
6
|
|
7
|
+
from runem.log import log
|
7
8
|
from runem.types import Config
|
8
9
|
|
9
10
|
CFG_FILE_YAML = pathlib.Path(".runem.yml")
|
@@ -47,7 +48,7 @@ def _find_cfg() -> pathlib.Path:
|
|
47
48
|
return cfg_candidate
|
48
49
|
|
49
50
|
# error out and exit as we currently require the cfg file as it lists jobs.
|
50
|
-
|
51
|
+
log(f"ERROR: Config not found! Looked from {start_dirs}")
|
51
52
|
sys.exit(1)
|
52
53
|
|
53
54
|
|
runem/config_parse.py
CHANGED
@@ -4,7 +4,8 @@ import typing
|
|
4
4
|
from collections import defaultdict
|
5
5
|
|
6
6
|
from runem.config_metadata import ConfigMetadata
|
7
|
-
from runem.
|
7
|
+
from runem.job_wrapper_python import get_job_wrapper
|
8
|
+
from runem.log import log
|
8
9
|
from runem.types import (
|
9
10
|
Config,
|
10
11
|
ConfigNodes,
|
@@ -73,12 +74,12 @@ def parse_job_config(
|
|
73
74
|
try:
|
74
75
|
job_names_used = job["label"] in in_out_job_names
|
75
76
|
if job_names_used:
|
76
|
-
|
77
|
-
|
77
|
+
log("ERROR: duplicate job label!")
|
78
|
+
log(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
|
78
79
|
sys.exit(1)
|
79
80
|
|
80
81
|
# try and load the function _before_ we schedule it's execution
|
81
|
-
|
82
|
+
get_job_wrapper(job, cfg_filepath)
|
82
83
|
phase_id: PhaseName = job["when"]["phase"]
|
83
84
|
in_out_jobs_by_phase[phase_id].append(job)
|
84
85
|
|
@@ -138,7 +139,7 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
|
|
138
139
|
)
|
139
140
|
|
140
141
|
if not phase_order:
|
141
|
-
|
142
|
+
log("WARNING: phase ordering not configured! Runs will be non-deterministic!")
|
142
143
|
phase_order = tuple(job_phases)
|
143
144
|
|
144
145
|
# tags = tags.union(("python", "es", "firebase_funcs"))
|
@@ -2,15 +2,17 @@ import inspect
|
|
2
2
|
import os
|
3
3
|
import pathlib
|
4
4
|
import typing
|
5
|
+
import uuid
|
5
6
|
from datetime import timedelta
|
6
7
|
from timeit import default_timer as timer
|
7
8
|
|
8
9
|
from runem.config_metadata import ConfigMetadata
|
9
|
-
from runem.
|
10
|
+
from runem.job_wrapper_python import get_job_wrapper
|
11
|
+
from runem.log import log
|
10
12
|
from runem.types import FilePathList, FilePathListLookup, JobConfig, JobReturn, JobTags
|
11
13
|
|
12
14
|
|
13
|
-
def
|
15
|
+
def job_execute_inner(
|
14
16
|
job_config: JobConfig,
|
15
17
|
config_metadata: ConfigMetadata,
|
16
18
|
file_lists: FilePathListLookup,
|
@@ -21,12 +23,12 @@ def job_runner_inner(
|
|
21
23
|
"""
|
22
24
|
label = job_config["label"]
|
23
25
|
if config_metadata.args.verbose:
|
24
|
-
|
26
|
+
log(f"START: {label}")
|
25
27
|
root_path: pathlib.Path = config_metadata.cfg_filepath.parent
|
26
28
|
function: typing.Callable
|
27
29
|
job_tags: JobTags = set(job_config["when"]["tags"])
|
28
30
|
os.chdir(root_path)
|
29
|
-
function =
|
31
|
+
function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
|
30
32
|
|
31
33
|
# get the files for all files found for this job's tags
|
32
34
|
file_list: FilePathList = []
|
@@ -36,7 +38,7 @@ def job_runner_inner(
|
|
36
38
|
|
37
39
|
if not file_list:
|
38
40
|
# no files to work on
|
39
|
-
|
41
|
+
log(f"WARNING: skipping job '{label}', no files for job")
|
40
42
|
return (f"{label}: no files!", timedelta(0)), None
|
41
43
|
|
42
44
|
if (
|
@@ -52,7 +54,7 @@ def job_runner_inner(
|
|
52
54
|
start = timer()
|
53
55
|
func_signature = inspect.signature(function)
|
54
56
|
if config_metadata.args.verbose:
|
55
|
-
|
57
|
+
log(f"job: running {job_config['label']}")
|
56
58
|
reports: JobReturn
|
57
59
|
if "args" in func_signature.parameters:
|
58
60
|
reports = function(config_metadata.args, config_metadata.options, file_list)
|
@@ -68,18 +70,23 @@ def job_runner_inner(
|
|
68
70
|
end = timer()
|
69
71
|
time_taken: timedelta = timedelta(seconds=end - start)
|
70
72
|
if config_metadata.args.verbose:
|
71
|
-
|
73
|
+
log(f"DONE: {label}: {time_taken}")
|
72
74
|
timing_data = (label, time_taken)
|
73
75
|
return (timing_data, reports)
|
74
76
|
|
75
77
|
|
76
|
-
def
|
78
|
+
def job_execute(
|
77
79
|
job_config: JobConfig,
|
80
|
+
running_jobs: typing.Dict[str, str],
|
78
81
|
config_metadata: ConfigMetadata,
|
79
82
|
file_lists: FilePathListLookup,
|
80
83
|
) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
|
81
|
-
"""
|
84
|
+
"""Thin-wrapper around job_execute_inner needed for mocking in tests.
|
82
85
|
|
83
86
|
Needed for faster tests.
|
84
87
|
"""
|
85
|
-
|
88
|
+
this_id: str = str(uuid.uuid4())
|
89
|
+
running_jobs[this_id] = job_config["label"]
|
90
|
+
results = job_execute_inner(job_config, config_metadata, file_lists)
|
91
|
+
del running_jobs[this_id]
|
92
|
+
return results
|
runem/job_filter.py
CHANGED
@@ -2,6 +2,7 @@ import typing
|
|
2
2
|
from collections import defaultdict
|
3
3
|
|
4
4
|
from runem.config_metadata import ConfigMetadata
|
5
|
+
from runem.log import log
|
5
6
|
from runem.types import (
|
6
7
|
JobConfig,
|
7
8
|
JobNames,
|
@@ -30,7 +31,7 @@ def _get_jobs_matching(
|
|
30
31
|
matching_tags = job_tags.intersection(tags)
|
31
32
|
if not matching_tags:
|
32
33
|
if verbose:
|
33
|
-
|
34
|
+
log(
|
34
35
|
(
|
35
36
|
f"not running job '{job['label']}' because it doesn't have "
|
36
37
|
f"any of the following tags: {printable_set(tags)}"
|
@@ -40,7 +41,7 @@ def _get_jobs_matching(
|
|
40
41
|
|
41
42
|
if job["label"] not in job_names:
|
42
43
|
if verbose:
|
43
|
-
|
44
|
+
log(
|
44
45
|
(
|
45
46
|
f"not running job '{job['label']}' because it isn't in the "
|
46
47
|
f"list of job names. See --jobs and --not-jobs"
|
@@ -51,7 +52,7 @@ def _get_jobs_matching(
|
|
51
52
|
has_tags_to_avoid = job_tags.intersection(tags_to_avoid)
|
52
53
|
if has_tags_to_avoid:
|
53
54
|
if verbose:
|
54
|
-
|
55
|
+
log(
|
55
56
|
(
|
56
57
|
f"not running job '{job['label']}' because it contains the "
|
57
58
|
f"following tags: {printable_set(has_tags_to_avoid)}"
|
@@ -73,17 +74,23 @@ def filter_jobs(
|
|
73
74
|
jobs: PhaseGroupedJobs = config_metadata.jobs
|
74
75
|
verbose: bool = config_metadata.args.verbose
|
75
76
|
if tags_to_run:
|
76
|
-
|
77
|
+
log(f"filtering for tags {printable_set(tags_to_run)}", decorate=True, end="")
|
77
78
|
if tags_to_avoid:
|
78
79
|
if tags_to_run:
|
79
|
-
|
80
|
-
|
80
|
+
log(", ", decorate=False, end="")
|
81
|
+
else:
|
82
|
+
log(decorate=True, end="")
|
83
|
+
log(
|
84
|
+
f"excluding jobs with tags {printable_set(tags_to_avoid)}",
|
85
|
+
decorate=False,
|
86
|
+
end="",
|
87
|
+
)
|
81
88
|
if tags_to_run or tags_to_avoid:
|
82
|
-
|
89
|
+
log(decorate=False)
|
83
90
|
filtered_jobs: PhaseGroupedJobs = defaultdict(list)
|
84
91
|
for phase in config_metadata.phases:
|
85
92
|
if phase not in phases_to_run:
|
86
|
-
|
93
|
+
log(f"skipping phase '{phase}'")
|
87
94
|
continue
|
88
95
|
_get_jobs_matching(
|
89
96
|
phase=phase,
|
@@ -95,10 +102,10 @@ def filter_jobs(
|
|
95
102
|
verbose=verbose,
|
96
103
|
)
|
97
104
|
if len(filtered_jobs[phase]) == 0:
|
98
|
-
|
105
|
+
log(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
|
99
106
|
continue
|
100
107
|
|
101
|
-
|
102
|
-
|
108
|
+
log((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
|
109
|
+
log(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
|
103
110
|
|
104
111
|
return filtered_jobs
|
@@ -20,20 +20,35 @@ def _load_python_function_from_module(
|
|
20
20
|
).absolute()
|
21
21
|
|
22
22
|
# load the function
|
23
|
-
|
24
|
-
|
23
|
+
try:
|
24
|
+
module_spec = module_spec_from_file_location(
|
25
|
+
function_to_load, abs_module_file_path
|
26
|
+
)
|
27
|
+
if not module_spec:
|
28
|
+
raise FileNotFoundError()
|
29
|
+
if not module_spec.loader:
|
30
|
+
raise FunctionNotFound("unable to load module")
|
31
|
+
except FileNotFoundError as err:
|
25
32
|
raise FunctionNotFound(
|
26
33
|
(
|
27
34
|
f"unable to load '{function_to_load}' from '{str(module_file_path)} "
|
28
35
|
f"relative to '{str(cfg_filepath)}"
|
29
36
|
)
|
30
|
-
)
|
37
|
+
) from err
|
31
38
|
|
32
39
|
module = module_from_spec(module_spec)
|
33
|
-
|
34
|
-
if not module_spec.loader:
|
40
|
+
if not module:
|
35
41
|
raise FunctionNotFound("unable to load module")
|
36
|
-
|
42
|
+
sys.modules[module_name] = module
|
43
|
+
try:
|
44
|
+
module_spec.loader.exec_module(module)
|
45
|
+
except FileNotFoundError as err:
|
46
|
+
raise FunctionNotFound(
|
47
|
+
(
|
48
|
+
f"unable to load '{function_to_load}' from '{str(module_file_path)} "
|
49
|
+
f"relative to '{str(cfg_filepath)}"
|
50
|
+
)
|
51
|
+
) from err
|
37
52
|
try:
|
38
53
|
function: JobFunction = getattr(module, function_to_load)
|
39
54
|
except AttributeError as err:
|
@@ -70,7 +85,7 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
|
|
70
85
|
return module_path.relative_to(cfg_filepath.parent.absolute())
|
71
86
|
|
72
87
|
|
73
|
-
def
|
88
|
+
def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
|
74
89
|
"""Given a job-description dynamically loads the job-function so we can call it.
|
75
90
|
|
76
91
|
Side-effects: also re-addressed the job-config.
|
runem/log.py
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
import typing
|
2
|
+
|
3
|
+
|
4
|
+
def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
|
5
|
+
"""Thin wrapper around 'print', so we can change the output.
|
6
|
+
|
7
|
+
One way we change it is to decorate the output with 'runem'
|
8
|
+
"""
|
9
|
+
if decorate:
|
10
|
+
msg = f"runem: {msg}"
|
11
|
+
print(msg, end=end)
|
runem/report.py
CHANGED
@@ -2,6 +2,7 @@ import typing
|
|
2
2
|
from collections import defaultdict
|
3
3
|
from datetime import timedelta
|
4
4
|
|
5
|
+
from runem.log import log
|
5
6
|
from runem.types import (
|
6
7
|
JobReturn,
|
7
8
|
JobRunMetadatasByPhase,
|
@@ -25,12 +26,15 @@ def _plot_times(
|
|
25
26
|
phase_run_oder: OrderedPhases,
|
26
27
|
timing_data: JobRunTimesByPhase,
|
27
28
|
) -> timedelta:
|
28
|
-
"""Prints a report to terminal on how well we performed.
|
29
|
+
"""Prints a report to terminal on how well we performed.
|
30
|
+
|
31
|
+
Also calculates the wall-clock time-saved for the user.
|
32
|
+
"""
|
29
33
|
labels: typing.List[str] = []
|
30
34
|
times: typing.List[float] = []
|
31
35
|
job_time_sum: timedelta = timedelta() # init to 0
|
32
36
|
for phase in phase_run_oder:
|
33
|
-
#
|
37
|
+
# log(f"Phase '{phase}' jobs took:")
|
34
38
|
phase_total_time: float = 0.0
|
35
39
|
phase_start_idx = len(labels)
|
36
40
|
for label, job_time in timing_data[phase]:
|
@@ -59,18 +63,37 @@ def _plot_times(
|
|
59
63
|
fig.show()
|
60
64
|
else: # pragma: FIXME: add code coverage
|
61
65
|
for label, time in zip(labels, times):
|
62
|
-
|
66
|
+
log(f"{label}: {time}s")
|
63
67
|
|
64
68
|
time_saved: timedelta = job_time_sum - overall_run_time
|
65
69
|
return time_saved
|
66
70
|
|
67
71
|
|
72
|
+
def _print_reports_by_phase(
|
73
|
+
phase_run_oder: OrderedPhases, report_data: JobRunReportByPhase
|
74
|
+
) -> None:
|
75
|
+
"""Logs out the reports by grouped by phase."""
|
76
|
+
for phase in phase_run_oder:
|
77
|
+
report_urls: ReportUrls = report_data[phase]
|
78
|
+
job_report_url_info: ReportUrlInfo
|
79
|
+
for job_report_url_info in report_urls:
|
80
|
+
if not job_report_url_info:
|
81
|
+
continue
|
82
|
+
log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
|
83
|
+
|
84
|
+
|
68
85
|
def report_on_run(
|
69
86
|
phase_run_oder: OrderedPhases,
|
70
87
|
job_run_metadatas: JobRunMetadatasByPhase,
|
71
88
|
overall_runtime: timedelta,
|
72
|
-
):
|
73
|
-
|
89
|
+
) -> timedelta:
|
90
|
+
"""Generate high-level reports AND prints out any reports returned by jobs.
|
91
|
+
|
92
|
+
IMPORTANT: returns the wall-clock time saved to the user.
|
93
|
+
"""
|
94
|
+
log("reports:")
|
95
|
+
|
96
|
+
# First, collate all data, timing and reports
|
74
97
|
timing_data: JobRunTimesByPhase = defaultdict(list)
|
75
98
|
report_data: JobRunReportByPhase = defaultdict(list)
|
76
99
|
phase: PhaseName
|
@@ -80,19 +103,21 @@ def report_on_run(
|
|
80
103
|
for timing, reports in job_run_metadatas[phase]:
|
81
104
|
timing_data[phase].append(timing)
|
82
105
|
if reports:
|
106
|
+
# the job returned some report urls, record them against the
|
107
|
+
# job's phase
|
83
108
|
report_data[phase].extend(reports["reportUrls"])
|
109
|
+
|
110
|
+
# Now plot the times on the terminal to give a visual report of the timing.
|
111
|
+
# Also, calculate the time saved by runem, a key selling-point metric
|
84
112
|
time_saved: timedelta = _plot_times(
|
85
113
|
overall_run_time=overall_runtime,
|
86
114
|
phase_run_oder=phase_run_oder,
|
87
115
|
timing_data=timing_data,
|
88
116
|
)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
print(
|
96
|
-
f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}"
|
97
|
-
)
|
117
|
+
|
118
|
+
# Penultimate-ly print out the available reports grouped by run-phase.
|
119
|
+
_print_reports_by_phase(phase_run_oder, report_data)
|
120
|
+
|
121
|
+
# Return the key metric for runem, the wall-clock time saved to the user
|
122
|
+
# TODO: write this to disk
|
98
123
|
return time_saved
|
runem/run_command.py
CHANGED
@@ -5,6 +5,8 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
|
|
5
5
|
from subprocess import CompletedProcess
|
6
6
|
from subprocess import run as subprocess_run
|
7
7
|
|
8
|
+
from runem.log import log
|
9
|
+
|
8
10
|
TERMINAL_WIDTH = 86
|
9
11
|
|
10
12
|
|
@@ -42,10 +44,10 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
42
44
|
"""Runs the given command, returning stdout or throwing on any error."""
|
43
45
|
cmd_string: str = " ".join(cmd)
|
44
46
|
if verbose:
|
45
|
-
|
47
|
+
log(f"running: start: {label}: {cmd_string}")
|
46
48
|
if valid_exit_ids is not None:
|
47
49
|
valid_exit_strs = ",".join([str(exit_code) for exit_code in valid_exit_ids])
|
48
|
-
|
50
|
+
log(f"\tallowed return ids are: {valid_exit_strs}")
|
49
51
|
|
50
52
|
if valid_exit_ids is None:
|
51
53
|
valid_exit_ids = (0,)
|
@@ -60,13 +62,13 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
60
62
|
run_env_as_string = " ".join(
|
61
63
|
[f"{key}='{value}'" for key, value in run_env.items()]
|
62
64
|
)
|
63
|
-
|
65
|
+
log(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
|
64
66
|
|
65
67
|
if env_overrides:
|
66
68
|
env_overrides_as_string = " ".join(
|
67
69
|
[f"{key}='{value}'" for key, value in env_overrides.items()]
|
68
70
|
)
|
69
|
-
|
71
|
+
log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
|
70
72
|
|
71
73
|
env_overrides_dict = {}
|
72
74
|
if env_overrides:
|
@@ -127,6 +129,6 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
127
129
|
assert process is not None
|
128
130
|
cmd_stdout: str = get_stdout(process, prefix=label)
|
129
131
|
if verbose:
|
130
|
-
|
131
|
-
|
132
|
+
log(cmd_stdout)
|
133
|
+
log(f"running: done: {label}: {cmd_string}")
|
132
134
|
return cmd_stdout
|
runem/runem.py
CHANGED
@@ -23,19 +23,24 @@ import multiprocessing
|
|
23
23
|
import os
|
24
24
|
import pathlib
|
25
25
|
import sys
|
26
|
+
import time
|
26
27
|
import typing
|
27
28
|
from collections import defaultdict
|
28
29
|
from datetime import timedelta
|
29
30
|
from itertools import repeat
|
31
|
+
from multiprocessing.managers import DictProxy, ValueProxy
|
30
32
|
from timeit import default_timer as timer
|
31
33
|
|
34
|
+
from halo import Halo
|
35
|
+
|
32
36
|
from runem.command_line import parse_args
|
33
37
|
from runem.config import load_config
|
34
38
|
from runem.config_metadata import ConfigMetadata
|
35
39
|
from runem.config_parse import parse_config
|
36
40
|
from runem.files import find_files
|
41
|
+
from runem.job_execute import job_execute
|
37
42
|
from runem.job_filter import filter_jobs
|
38
|
-
from runem.
|
43
|
+
from runem.log import log
|
39
44
|
from runem.report import report_on_run
|
40
45
|
from runem.types import (
|
41
46
|
Config,
|
@@ -69,11 +74,27 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
|
|
69
74
|
config_metadata = parse_args(config_metadata, argv)
|
70
75
|
|
71
76
|
if config_metadata.args.verbose:
|
72
|
-
|
77
|
+
log(f"loaded config from {cfg_filepath}")
|
73
78
|
|
74
79
|
return config_metadata
|
75
80
|
|
76
81
|
|
82
|
+
def _progress_updater(
|
83
|
+
label: str, running_jobs: typing.Dict[str, str], is_running: ValueProxy
|
84
|
+
) -> None:
|
85
|
+
spinner = Halo(text="", spinner="dots")
|
86
|
+
spinner.start()
|
87
|
+
|
88
|
+
while is_running.value:
|
89
|
+
running_job_names: typing.List[str] = [
|
90
|
+
f"'{job}'" for job in sorted(list(running_jobs.values()))
|
91
|
+
]
|
92
|
+
printable_jobs: str = ", ".join(running_job_names)
|
93
|
+
spinner.text = f"{label}: {printable_jobs}"
|
94
|
+
time.sleep(0.1)
|
95
|
+
spinner.stop()
|
96
|
+
|
97
|
+
|
77
98
|
def _process_jobs(
|
78
99
|
config_metadata: ConfigMetadata,
|
79
100
|
file_lists: FilePathListLookup,
|
@@ -96,22 +117,38 @@ def _process_jobs(
|
|
96
117
|
else multiprocessing.cpu_count()
|
97
118
|
)
|
98
119
|
num_concurrent_procs = min(num_concurrent_procs, len(jobs))
|
99
|
-
|
120
|
+
log(
|
100
121
|
(
|
101
122
|
f"Running '{phase}' with {num_concurrent_procs} workers "
|
102
123
|
f"processing {len(jobs)} jobs"
|
103
124
|
)
|
104
125
|
)
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
repeat(file_lists),
|
113
|
-
),
|
126
|
+
|
127
|
+
with multiprocessing.Manager() as manager:
|
128
|
+
running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
|
129
|
+
is_running: ValueProxy = manager.Value("b", True)
|
130
|
+
|
131
|
+
terminal_writer_process = multiprocessing.Process(
|
132
|
+
target=_progress_updater, args=(phase, running_jobs, is_running)
|
114
133
|
)
|
134
|
+
terminal_writer_process.start()
|
135
|
+
|
136
|
+
try:
|
137
|
+
with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
|
138
|
+
# use starmap so we can pass down the job-configs and the args and the files
|
139
|
+
in_out_job_run_metadatas[phase] = pool.starmap(
|
140
|
+
job_execute,
|
141
|
+
zip(
|
142
|
+
jobs,
|
143
|
+
repeat(running_jobs),
|
144
|
+
repeat(config_metadata),
|
145
|
+
repeat(file_lists),
|
146
|
+
),
|
147
|
+
)
|
148
|
+
finally:
|
149
|
+
# Signal the terminal_writer process to exit
|
150
|
+
is_running.value = False
|
151
|
+
terminal_writer_process.join()
|
115
152
|
|
116
153
|
|
117
154
|
def _process_jobs_by_phase(
|
@@ -139,7 +176,7 @@ def _process_jobs_by_phase(
|
|
139
176
|
continue
|
140
177
|
|
141
178
|
if config_metadata.args.verbose:
|
142
|
-
|
179
|
+
log(f"Running Phase {phase}")
|
143
180
|
|
144
181
|
_process_jobs(
|
145
182
|
config_metadata, file_lists, in_out_job_run_metadatas, phase, jobs
|
@@ -158,11 +195,11 @@ def _main(
|
|
158
195
|
|
159
196
|
file_lists: FilePathListLookup = find_files(config_metadata)
|
160
197
|
assert file_lists
|
161
|
-
|
198
|
+
log(f"found {len(file_lists)} batches, ", end="")
|
162
199
|
for tag in sorted(file_lists.keys()):
|
163
200
|
file_list = file_lists[tag]
|
164
|
-
|
165
|
-
|
201
|
+
log(f"{len(file_list)} '{tag}' files, ", decorate=False, end="")
|
202
|
+
log(decorate=False) # new line
|
166
203
|
|
167
204
|
filtered_jobs_by_phase: PhaseGroupedJobs = filter_jobs(
|
168
205
|
config_metadata=config_metadata,
|
@@ -202,7 +239,7 @@ def timed_main(argv: typing.List[str]) -> None:
|
|
202
239
|
end = timer()
|
203
240
|
time_taken: timedelta = timedelta(seconds=end - start)
|
204
241
|
time_saved = report_on_run(phase_run_oder, job_run_metadatas, time_taken)
|
205
|
-
|
242
|
+
log(
|
206
243
|
(
|
207
244
|
f"DONE: runem took: {time_taken.total_seconds()}s, "
|
208
245
|
f"saving you {time_saved.total_seconds()}s"
|
runem/types.py
CHANGED
@@ -18,8 +18,9 @@ JobPhases = typing.Set[str]
|
|
18
18
|
JobTags = typing.Set[JobTag]
|
19
19
|
PhaseName = str
|
20
20
|
OrderedPhases = typing.Tuple[PhaseName, ...]
|
21
|
-
|
22
|
-
|
21
|
+
ReportName = str
|
22
|
+
ReportUrl = typing.Union[str, pathlib.Path]
|
23
|
+
ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
|
23
24
|
ReportUrls = typing.List[ReportUrlInfo]
|
24
25
|
|
25
26
|
|
@@ -1,11 +1,13 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: runem
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.18
|
4
4
|
Summary: Awesome runem created by lursight
|
5
5
|
Home-page: https://github.com/lursight/runem/
|
6
6
|
Author: lursight
|
7
7
|
Description-Content-Type: text/markdown
|
8
8
|
License-File: LICENSE
|
9
|
+
Requires-Dist: halo
|
10
|
+
Requires-Dist: PyYAML
|
9
11
|
Provides-Extra: test
|
10
12
|
Requires-Dist: black ==23.11.0 ; extra == 'test'
|
11
13
|
Requires-Dist: coverage ==7.3.2 ; extra == 'test'
|
@@ -23,7 +25,6 @@ Requires-Dist: pytest-cov ==4.1.0 ; extra == 'test'
|
|
23
25
|
Requires-Dist: pytest-profiling ==1.7.0 ; extra == 'test'
|
24
26
|
Requires-Dist: pytest-xdist ==3.3.1 ; extra == 'test'
|
25
27
|
Requires-Dist: pytest ==7.4.3 ; extra == 'test'
|
26
|
-
Requires-Dist: PyYAML ==6.0.1 ; extra == 'test'
|
27
28
|
Requires-Dist: termplotlib ==0.3.9 ; extra == 'test'
|
28
29
|
Requires-Dist: types-PyYAML ==6.0.12.12 ; extra == 'test'
|
29
30
|
Requires-Dist: requests-mock ==1.10.0 ; extra == 'test'
|
@@ -0,0 +1,25 @@
|
|
1
|
+
runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
|
3
|
+
runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
|
4
|
+
runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
|
5
|
+
runem/command_line.py,sha256=lb1tvYOEVYpB_bUTRL8KxjLFuV4vKn8btFKm-28GMu0,9464
|
6
|
+
runem/config.py,sha256=h0r2WYUlAgdokUS1VU1sjFHgrN_cEq339ph4Kn1IBBM,1937
|
7
|
+
runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
|
8
|
+
runem/config_parse.py,sha256=gso5Lziw8yII5jGSx8fR-IAs7nxUzhzitnRAEXsg5f8,5157
|
9
|
+
runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
|
10
|
+
runem/job_execute.py,sha256=x-IGTG5KYnTJtCYm4jP8_WOQ_UM_BAiG5b3UUKp527A,3042
|
11
|
+
runem/job_filter.py,sha256=nltSRK1IKp2fuTCHOZFcii0FBL-R3RKvrnyWXTPjoMw,3478
|
12
|
+
runem/job_wrapper_python.py,sha256=TG9bzDe1dgTnsrXrCPd6LzN8qbMkH4rEv_vXC_zlJc0,4226
|
13
|
+
runem/log.py,sha256=m1lI4V8ygM53pY4Go4eEzvEJY8srVoItxUNhdp_Vrqg,314
|
14
|
+
runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
|
+
runem/report.py,sha256=AyD2yT_8ODo5QRuFkVFIecp6ycT4XElJbleWviZbCfI,4032
|
16
|
+
runem/run_command.py,sha256=6QxH8pb4Lug-d0MCP2lBKq2TJlQloxRlq0hYJM6IScc,4528
|
17
|
+
runem/runem.py,sha256=Q5ryZtArAGuTlHF07aobJkRIzurKIUsMSUPzEksL6LY,8227
|
18
|
+
runem/types.py,sha256=Hw7G7ut7yNaoc3XiDNKeM-cAU-DDu3fDOne13FrutS0,5374
|
19
|
+
runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
|
20
|
+
runem-0.0.18.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
21
|
+
runem-0.0.18.dist-info/METADATA,sha256=VzkG4gPaopCtwCnEx1XRGLk9iNWpLPjl4x8BY-JvSSA,15909
|
22
|
+
runem-0.0.18.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
23
|
+
runem-0.0.18.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
|
24
|
+
runem-0.0.18.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
|
25
|
+
runem-0.0.18.dist-info/RECORD,,
|
runem-0.0.16.dist-info/RECORD
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
-
runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
|
3
|
-
runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
|
4
|
-
runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
|
5
|
-
runem/command_line.py,sha256=MD3tAdXO7WNKRabwSJwBks1az2mYyTAxifxX-vGA8yc,9425
|
6
|
-
runem/config.py,sha256=jtpMOS-7J0YFkQmXpDy3CPtohEd4HBHGmjPTzkL0Tyw,1913
|
7
|
-
runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
|
8
|
-
runem/config_parse.py,sha256=eA5oAW8T7-kv6TnNEsnhIvRP0AoPItufwaTTaMr7yZ0,5140
|
9
|
-
runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
|
10
|
-
runem/job_filter.py,sha256=cNlc9UVi6S1O9Ui95GH3n4dFEYRiGJ4KUIt5xPDQWWw,3313
|
11
|
-
runem/job_function_python.py,sha256=iNRKQp0Rlkh1VektOxUSa_KqrrY0yB7jDChcRt-s6oQ,3754
|
12
|
-
runem/job_runner.py,sha256=cCpZg_1LOMalQoY23_5A7R8vu7aaNjliZL4q8qxdIz0,2834
|
13
|
-
runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
|
-
runem/report.py,sha256=S-KV70gMBUlxjqtzQkCqSIDhsRk6X20i9RKLv6-Y5vk,3104
|
15
|
-
runem/run_command.py,sha256=UPwQU7eC5FADcX3aPrNP8kp-g5YONBdwwjsz-NCaMno,4533
|
16
|
-
runem/runem.py,sha256=LEccvElpxpuKM6Bl4gvrc1-ZlMkSB8b7K13l5nyzNP4,6941
|
17
|
-
runem/types.py,sha256=IxUDGHOQZUc86OPjDHDKwrcBFr78v-MNvTpPwUzWoaI,5337
|
18
|
-
runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
|
19
|
-
runem-0.0.16.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
20
|
-
runem-0.0.16.dist-info/METADATA,sha256=P-__qoOFtuC8JZU7uh3hEBeN_U7kHW7U8hGY9q_5A6U,15915
|
21
|
-
runem-0.0.16.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
22
|
-
runem-0.0.16.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
|
23
|
-
runem-0.0.16.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
|
24
|
-
runem-0.0.16.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|