runem 0.0.16__tar.gz → 0.0.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {runem-0.0.16 → runem-0.0.18}/HISTORY.md +114 -0
- {runem-0.0.16 → runem-0.0.18}/PKG-INFO +3 -2
- runem-0.0.18/runem/VERSION +1 -0
- runem-0.0.18/runem/cli/initialise_options.py +25 -0
- {runem-0.0.16 → runem-0.0.18}/runem/command_line.py +7 -6
- {runem-0.0.16 → runem-0.0.18}/runem/config.py +2 -1
- {runem-0.0.16 → runem-0.0.18}/runem/config_parse.py +6 -5
- runem-0.0.16/runem/job_runner.py → runem-0.0.18/runem/job_execute.py +17 -10
- {runem-0.0.16 → runem-0.0.18}/runem/job_filter.py +18 -11
- runem-0.0.16/runem/job_function_python.py → runem-0.0.18/runem/job_wrapper_python.py +22 -7
- runem-0.0.18/runem/log.py +11 -0
- {runem-0.0.16 → runem-0.0.18}/runem/report.py +39 -14
- {runem-0.0.16 → runem-0.0.18}/runem/run_command.py +8 -6
- {runem-0.0.16 → runem-0.0.18}/runem/runem.py +54 -17
- {runem-0.0.16 → runem-0.0.18}/runem/types.py +3 -2
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/PKG-INFO +3 -2
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/SOURCES.txt +7 -4
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/requires.txt +2 -1
- runem-0.0.18/tests/cli/test_initialise_options.py +101 -0
- {runem-0.0.16 → runem-0.0.18}/tests/data/help_output.txt +13 -15
- {runem-0.0.16 → runem-0.0.18}/tests/test_config_parse.py +42 -0
- runem-0.0.16/tests/test_job_runner.py → runem-0.0.18/tests/test_job_execute.py +11 -11
- runem-0.0.16/tests/test_job_function_python.py → runem-0.0.18/tests/test_job_wrapper_python.py +86 -9
- {runem-0.0.16 → runem-0.0.18}/tests/test_report.py +41 -3
- {runem-0.0.16 → runem-0.0.18}/tests/test_run_command.py +8 -8
- {runem-0.0.16 → runem-0.0.18}/tests/test_runem.py +212 -120
- runem-0.0.16/runem/VERSION +0 -1
- {runem-0.0.16 → runem-0.0.18}/Containerfile +0 -0
- {runem-0.0.16 → runem-0.0.18}/LICENSE +0 -0
- {runem-0.0.16 → runem-0.0.18}/MANIFEST.in +0 -0
- {runem-0.0.16 → runem-0.0.18}/README.md +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/__init__.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/__main__.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/base.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/cli.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/config_metadata.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/files.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/py.typed +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem/utils.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/dependency_links.txt +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/entry_points.txt +0 -0
- {runem-0.0.16 → runem-0.0.18}/runem.egg-info/top_level.txt +0 -0
- {runem-0.0.16 → runem-0.0.18}/setup.cfg +0 -0
- {runem-0.0.16 → runem-0.0.18}/setup.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/__init__.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/conftest.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/test_base.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/test_cli.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/test_config.py +0 -0
- {runem-0.0.16 → runem-0.0.18}/tests/test_files.py +0 -0
@@ -4,6 +4,120 @@ Changelog
|
|
4
4
|
|
5
5
|
(unreleased)
|
6
6
|
------------
|
7
|
+
- Merge pull request #9 from lursight/chore/rename_core_functions.
|
8
|
+
[Frank Harrison]
|
9
|
+
|
10
|
+
Chore/rename core functions
|
11
|
+
- Chore(core-func-rename): renames functions and fixes imports. [Frank
|
12
|
+
Harrison]
|
13
|
+
- Chore(core-func-rename): renames files to better reflect contents.
|
14
|
+
[Frank Harrison]
|
15
|
+
- Merge pull request #8 from lursight/chore/more_typing_strictness.
|
16
|
+
[Frank Harrison]
|
17
|
+
|
18
|
+
Chore/more typing strictness
|
19
|
+
- Chore(mypy-strict): switches mypy to use strict-mode. [Frank Harrison]
|
20
|
+
|
21
|
+
This should catch more issues down the line
|
22
|
+
- Chore(mypy-strict): enable disallow_untyped_calls and annotate it.
|
23
|
+
[Frank Harrison]
|
24
|
+
- Chore(mypy-strict): enable disallow_untyped_defs in mypy. [Frank
|
25
|
+
Harrison]
|
26
|
+
- Chore(mypy-strict): enables check_untyped_defs in mypy. [Frank
|
27
|
+
Harrison]
|
28
|
+
- Chore(mypy-strict): annotates mypy config options. [Frank Harrison]
|
29
|
+
- Merge pull request #7 from lursight/chore/project_status. [Frank
|
30
|
+
Harrison]
|
31
|
+
|
32
|
+
chore(project-status): removes the project rename from actions
|
33
|
+
- Chore(project-status): removes the project rename from actions. [Frank
|
34
|
+
Harrison]
|
35
|
+
- Merge pull request #5 from lursight/feat/job_spinner. [Frank Harrison]
|
36
|
+
|
37
|
+
Feat/job spinner
|
38
|
+
- Feat(progress): gets a progress spinner working. [Frank Harrison]
|
39
|
+
- Feat(progress): adds way to track running jobs for multip-proc jobs.
|
40
|
+
[Frank Harrison]
|
41
|
+
- Merge pull request #3 from
|
42
|
+
lursight/dependabot/github_actions/actions/setup-python-5. [Frank
|
43
|
+
Harrison]
|
44
|
+
|
45
|
+
chore(deps): bump actions/setup-python from 4 to 5
|
46
|
+
- Chore(deps): bump actions/setup-python from 4 to 5. [dependabot[bot]]
|
47
|
+
|
48
|
+
Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
|
49
|
+
- [Release notes](https://github.com/actions/setup-python/releases)
|
50
|
+
- [Commits](https://github.com/actions/setup-python/compare/v4...v5)
|
51
|
+
|
52
|
+
---
|
53
|
+
updated-dependencies:
|
54
|
+
- dependency-name: actions/setup-python
|
55
|
+
dependency-type: direct:production
|
56
|
+
update-type: version-update:semver-major
|
57
|
+
...
|
58
|
+
- Merge pull request #6 from lursight/fix/help_tests. [Frank Harrison]
|
59
|
+
|
60
|
+
Fix/help tests
|
61
|
+
- Chore(fix-help-tests): fixing the help-text test in ci/cd. [Frank
|
62
|
+
Harrison]
|
63
|
+
- Chore(pytest): stops reporting of coverage on test failures. [Frank
|
64
|
+
Harrison]
|
65
|
+
- Chore(pytest): stops failing coverage BEFORE we want to. [Frank
|
66
|
+
Harrison]
|
67
|
+
- Chore(type): fixes type errors. [Frank Harrison]
|
68
|
+
- Merge pull request #4 from lursight/fix/python3.10_support. [Frank
|
69
|
+
Harrison]
|
70
|
+
|
71
|
+
fix(python3.10): removes another pipe shortcut
|
72
|
+
- Fix(python3.10): removes another pipe shortcut. [Frank Harrison]
|
73
|
+
- Merge branch 'fix/python3.10_support' [Frank Harrison]
|
74
|
+
- Fix(python3.10): removes newer typing syntax. [Frank Harrison]
|
75
|
+
- Merge branch 'fix/coverage' [Frank Harrison]
|
76
|
+
- Fix(coverage): adds more coverage to parse_config() [Frank Harrison]
|
77
|
+
|
78
|
+
... specifically the bit that warns if we have to nordered phases
|
79
|
+
- Fix(coverage): adds more coverage to
|
80
|
+
_load_python_function_from_module() [Frank Harrison]
|
81
|
+
- Fix(coverage): adds more coverage to initialise_options() [Frank
|
82
|
+
Harrison]
|
83
|
+
- Fix(coverage): adds more coverage to report.py. [Frank Harrison]
|
84
|
+
- Fix(coverage): annotates a file that needs more coverage. [Frank
|
85
|
+
Harrison]
|
86
|
+
- Merge branch 'fix/spell_check' [Frank Harrison]
|
87
|
+
- Fix(spell-check): fixes the spell-checker by ignoring the history
|
88
|
+
file. [Frank Harrison]
|
89
|
+
|
90
|
+
... which contains typos in the commit hitsory
|
91
|
+
|
92
|
+
|
93
|
+
0.0.17 (2023-12-09)
|
94
|
+
-------------------
|
95
|
+
- Release: version 0.0.17 🚀 [Frank Harrison]
|
96
|
+
- Merge branch 'chore/run_in_check_mode_on_release' [Frank Harrison]
|
97
|
+
- Chore(release-checks): makes each command fail on 'make release'
|
98
|
+
[Frank Harrison]
|
99
|
+
- Chore(release-checks): run runem in check mode on 'make release'
|
100
|
+
[Frank Harrison]
|
101
|
+
- Merge branch 'fix/pyyaml_dep' [Frank Harrison]
|
102
|
+
- Fix(deps): adds the py-yaml dep to release requirements. [Frank
|
103
|
+
Harrison]
|
104
|
+
- Merge branch 'chore/log_output' [Frank Harrison]
|
105
|
+
- Chore(coverage): fixes up coverage, for now. [Frank Harrison]
|
106
|
+
- Fixup: logs. [Frank Harrison]
|
107
|
+
- Chore(black): formats log.py. [Frank Harrison]
|
108
|
+
- Chore(log-format): replaces print() with log. [Frank Harrison]
|
109
|
+
|
110
|
+
... also adds a prefix to the logging
|
111
|
+
- Merge branch 'chore/skip_spell_check_history' [Frank Harrison]
|
112
|
+
- Chore(spell-history): add cSpell:disable to HISTORY.md's frontmatter.
|
113
|
+
[Frank Harrison]
|
114
|
+
|
115
|
+
... because some of the commit messages contain spelling typos
|
116
|
+
|
117
|
+
|
118
|
+
0.0.16 (2023-12-05)
|
119
|
+
-------------------
|
120
|
+
- Release: version 0.0.16 🚀 [Frank Harrison]
|
7
121
|
- Merge branch 'chore/get_release_running_tests' [Frank Harrison]
|
8
122
|
- Chore(test-on-release): prints existing tags on make release. [Frank
|
9
123
|
Harrison]
|
@@ -1,11 +1,13 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: runem
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.18
|
4
4
|
Summary: Awesome runem created by lursight
|
5
5
|
Home-page: https://github.com/lursight/runem/
|
6
6
|
Author: lursight
|
7
7
|
Description-Content-Type: text/markdown
|
8
8
|
License-File: LICENSE
|
9
|
+
Requires-Dist: halo
|
10
|
+
Requires-Dist: PyYAML
|
9
11
|
Provides-Extra: test
|
10
12
|
Requires-Dist: black==23.11.0; extra == "test"
|
11
13
|
Requires-Dist: coverage==7.3.2; extra == "test"
|
@@ -23,7 +25,6 @@ Requires-Dist: pytest-cov==4.1.0; extra == "test"
|
|
23
25
|
Requires-Dist: pytest-profiling==1.7.0; extra == "test"
|
24
26
|
Requires-Dist: pytest-xdist==3.3.1; extra == "test"
|
25
27
|
Requires-Dist: pytest==7.4.3; extra == "test"
|
26
|
-
Requires-Dist: PyYAML==6.0.1; extra == "test"
|
27
28
|
Requires-Dist: termplotlib==0.3.9; extra == "test"
|
28
29
|
Requires-Dist: types-PyYAML==6.0.12.12; extra == "test"
|
29
30
|
Requires-Dist: requests-mock==1.10.0; extra == "test"
|
@@ -0,0 +1 @@
|
|
1
|
+
0.0.18
|
@@ -0,0 +1,25 @@
|
|
1
|
+
import argparse
|
2
|
+
|
3
|
+
from runem.config_metadata import ConfigMetadata
|
4
|
+
from runem.types import Options
|
5
|
+
|
6
|
+
|
7
|
+
def initialise_options(
|
8
|
+
config_metadata: ConfigMetadata,
|
9
|
+
args: argparse.Namespace,
|
10
|
+
) -> Options:
|
11
|
+
"""Initialises and returns the set of options to use for this run.
|
12
|
+
|
13
|
+
Returns the options dictionary
|
14
|
+
"""
|
15
|
+
|
16
|
+
options: Options = {
|
17
|
+
option["name"]: option["default"] for option in config_metadata.options_config
|
18
|
+
}
|
19
|
+
if config_metadata.options_config and args.overrides_on: # pragma: no branch
|
20
|
+
for option_name in args.overrides_on: # pragma: no branch
|
21
|
+
options[option_name] = True
|
22
|
+
if config_metadata.options_config and args.overrides_off: # pragma: no branch
|
23
|
+
for option_name in args.overrides_off:
|
24
|
+
options[option_name] = False
|
25
|
+
return options
|
@@ -5,6 +5,7 @@ import sys
|
|
5
5
|
import typing
|
6
6
|
|
7
7
|
from runem.config_metadata import ConfigMetadata
|
8
|
+
from runem.log import log
|
8
9
|
from runem.types import JobNames, OptionConfig, Options
|
9
10
|
from runem.utils import printable_set
|
10
11
|
|
@@ -146,7 +147,7 @@ def parse_args(
|
|
146
147
|
|
147
148
|
args = parser.parse_args(argv[1:])
|
148
149
|
|
149
|
-
options: Options =
|
150
|
+
options: Options = initialise_options(config_metadata, args)
|
150
151
|
|
151
152
|
if not _validate_filters(config_metadata, args):
|
152
153
|
sys.exit(1)
|
@@ -176,7 +177,7 @@ def _validate_filters(
|
|
176
177
|
for name, name_list in (("--jobs", args.jobs), ("--not-jobs", args.jobs_excluded)):
|
177
178
|
for job_name in name_list:
|
178
179
|
if job_name not in config_metadata.all_job_names:
|
179
|
-
|
180
|
+
log(
|
180
181
|
(
|
181
182
|
f"ERROR: invalid job-name '{job_name}' for {name}, "
|
182
183
|
f"choose from one of {printable_set(config_metadata.all_job_names)}"
|
@@ -188,7 +189,7 @@ def _validate_filters(
|
|
188
189
|
for name, tag_list in (("--tags", args.tags), ("--not-tags", args.tags_excluded)):
|
189
190
|
for tag in tag_list:
|
190
191
|
if tag not in config_metadata.all_job_tags:
|
191
|
-
|
192
|
+
log(
|
192
193
|
(
|
193
194
|
f"ERROR: invalid tag '{tag}' for {name}, "
|
194
195
|
f"choose from one of {printable_set(config_metadata.all_job_tags)}"
|
@@ -203,7 +204,7 @@ def _validate_filters(
|
|
203
204
|
):
|
204
205
|
for phase in phase_list:
|
205
206
|
if phase not in config_metadata.all_job_phases:
|
206
|
-
|
207
|
+
log(
|
207
208
|
(
|
208
209
|
f"ERROR: invalid phase '{phase}' for {name}, "
|
209
210
|
f"choose from one of {printable_set(config_metadata.all_job_phases)}"
|
@@ -213,7 +214,7 @@ def _validate_filters(
|
|
213
214
|
return True
|
214
215
|
|
215
216
|
|
216
|
-
def
|
217
|
+
def initialise_options(
|
217
218
|
config_metadata: ConfigMetadata,
|
218
219
|
args: argparse.Namespace,
|
219
220
|
) -> Options:
|
@@ -226,7 +227,7 @@ def _initialise_options(
|
|
226
227
|
option["name"]: option["default"] for option in config_metadata.options_config
|
227
228
|
}
|
228
229
|
if config_metadata.options_config and args.overrides_on: # pragma: no branch
|
229
|
-
for option_name in args.overrides_on:
|
230
|
+
for option_name in args.overrides_on: # pragma: no branch
|
230
231
|
options[option_name] = True
|
231
232
|
if config_metadata.options_config and args.overrides_off: # pragma: no branch
|
232
233
|
for option_name in args.overrides_off:
|
@@ -4,6 +4,7 @@ import typing
|
|
4
4
|
|
5
5
|
import yaml
|
6
6
|
|
7
|
+
from runem.log import log
|
7
8
|
from runem.types import Config
|
8
9
|
|
9
10
|
CFG_FILE_YAML = pathlib.Path(".runem.yml")
|
@@ -47,7 +48,7 @@ def _find_cfg() -> pathlib.Path:
|
|
47
48
|
return cfg_candidate
|
48
49
|
|
49
50
|
# error out and exit as we currently require the cfg file as it lists jobs.
|
50
|
-
|
51
|
+
log(f"ERROR: Config not found! Looked from {start_dirs}")
|
51
52
|
sys.exit(1)
|
52
53
|
|
53
54
|
|
@@ -4,7 +4,8 @@ import typing
|
|
4
4
|
from collections import defaultdict
|
5
5
|
|
6
6
|
from runem.config_metadata import ConfigMetadata
|
7
|
-
from runem.
|
7
|
+
from runem.job_wrapper_python import get_job_wrapper
|
8
|
+
from runem.log import log
|
8
9
|
from runem.types import (
|
9
10
|
Config,
|
10
11
|
ConfigNodes,
|
@@ -73,12 +74,12 @@ def parse_job_config(
|
|
73
74
|
try:
|
74
75
|
job_names_used = job["label"] in in_out_job_names
|
75
76
|
if job_names_used:
|
76
|
-
|
77
|
-
|
77
|
+
log("ERROR: duplicate job label!")
|
78
|
+
log(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
|
78
79
|
sys.exit(1)
|
79
80
|
|
80
81
|
# try and load the function _before_ we schedule it's execution
|
81
|
-
|
82
|
+
get_job_wrapper(job, cfg_filepath)
|
82
83
|
phase_id: PhaseName = job["when"]["phase"]
|
83
84
|
in_out_jobs_by_phase[phase_id].append(job)
|
84
85
|
|
@@ -138,7 +139,7 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
|
|
138
139
|
)
|
139
140
|
|
140
141
|
if not phase_order:
|
141
|
-
|
142
|
+
log("WARNING: phase ordering not configured! Runs will be non-deterministic!")
|
142
143
|
phase_order = tuple(job_phases)
|
143
144
|
|
144
145
|
# tags = tags.union(("python", "es", "firebase_funcs"))
|
@@ -2,15 +2,17 @@ import inspect
|
|
2
2
|
import os
|
3
3
|
import pathlib
|
4
4
|
import typing
|
5
|
+
import uuid
|
5
6
|
from datetime import timedelta
|
6
7
|
from timeit import default_timer as timer
|
7
8
|
|
8
9
|
from runem.config_metadata import ConfigMetadata
|
9
|
-
from runem.
|
10
|
+
from runem.job_wrapper_python import get_job_wrapper
|
11
|
+
from runem.log import log
|
10
12
|
from runem.types import FilePathList, FilePathListLookup, JobConfig, JobReturn, JobTags
|
11
13
|
|
12
14
|
|
13
|
-
def
|
15
|
+
def job_execute_inner(
|
14
16
|
job_config: JobConfig,
|
15
17
|
config_metadata: ConfigMetadata,
|
16
18
|
file_lists: FilePathListLookup,
|
@@ -21,12 +23,12 @@ def job_runner_inner(
|
|
21
23
|
"""
|
22
24
|
label = job_config["label"]
|
23
25
|
if config_metadata.args.verbose:
|
24
|
-
|
26
|
+
log(f"START: {label}")
|
25
27
|
root_path: pathlib.Path = config_metadata.cfg_filepath.parent
|
26
28
|
function: typing.Callable
|
27
29
|
job_tags: JobTags = set(job_config["when"]["tags"])
|
28
30
|
os.chdir(root_path)
|
29
|
-
function =
|
31
|
+
function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
|
30
32
|
|
31
33
|
# get the files for all files found for this job's tags
|
32
34
|
file_list: FilePathList = []
|
@@ -36,7 +38,7 @@ def job_runner_inner(
|
|
36
38
|
|
37
39
|
if not file_list:
|
38
40
|
# no files to work on
|
39
|
-
|
41
|
+
log(f"WARNING: skipping job '{label}', no files for job")
|
40
42
|
return (f"{label}: no files!", timedelta(0)), None
|
41
43
|
|
42
44
|
if (
|
@@ -52,7 +54,7 @@ def job_runner_inner(
|
|
52
54
|
start = timer()
|
53
55
|
func_signature = inspect.signature(function)
|
54
56
|
if config_metadata.args.verbose:
|
55
|
-
|
57
|
+
log(f"job: running {job_config['label']}")
|
56
58
|
reports: JobReturn
|
57
59
|
if "args" in func_signature.parameters:
|
58
60
|
reports = function(config_metadata.args, config_metadata.options, file_list)
|
@@ -68,18 +70,23 @@ def job_runner_inner(
|
|
68
70
|
end = timer()
|
69
71
|
time_taken: timedelta = timedelta(seconds=end - start)
|
70
72
|
if config_metadata.args.verbose:
|
71
|
-
|
73
|
+
log(f"DONE: {label}: {time_taken}")
|
72
74
|
timing_data = (label, time_taken)
|
73
75
|
return (timing_data, reports)
|
74
76
|
|
75
77
|
|
76
|
-
def
|
78
|
+
def job_execute(
|
77
79
|
job_config: JobConfig,
|
80
|
+
running_jobs: typing.Dict[str, str],
|
78
81
|
config_metadata: ConfigMetadata,
|
79
82
|
file_lists: FilePathListLookup,
|
80
83
|
) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
|
81
|
-
"""
|
84
|
+
"""Thin-wrapper around job_execute_inner needed for mocking in tests.
|
82
85
|
|
83
86
|
Needed for faster tests.
|
84
87
|
"""
|
85
|
-
|
88
|
+
this_id: str = str(uuid.uuid4())
|
89
|
+
running_jobs[this_id] = job_config["label"]
|
90
|
+
results = job_execute_inner(job_config, config_metadata, file_lists)
|
91
|
+
del running_jobs[this_id]
|
92
|
+
return results
|
@@ -2,6 +2,7 @@ import typing
|
|
2
2
|
from collections import defaultdict
|
3
3
|
|
4
4
|
from runem.config_metadata import ConfigMetadata
|
5
|
+
from runem.log import log
|
5
6
|
from runem.types import (
|
6
7
|
JobConfig,
|
7
8
|
JobNames,
|
@@ -30,7 +31,7 @@ def _get_jobs_matching(
|
|
30
31
|
matching_tags = job_tags.intersection(tags)
|
31
32
|
if not matching_tags:
|
32
33
|
if verbose:
|
33
|
-
|
34
|
+
log(
|
34
35
|
(
|
35
36
|
f"not running job '{job['label']}' because it doesn't have "
|
36
37
|
f"any of the following tags: {printable_set(tags)}"
|
@@ -40,7 +41,7 @@ def _get_jobs_matching(
|
|
40
41
|
|
41
42
|
if job["label"] not in job_names:
|
42
43
|
if verbose:
|
43
|
-
|
44
|
+
log(
|
44
45
|
(
|
45
46
|
f"not running job '{job['label']}' because it isn't in the "
|
46
47
|
f"list of job names. See --jobs and --not-jobs"
|
@@ -51,7 +52,7 @@ def _get_jobs_matching(
|
|
51
52
|
has_tags_to_avoid = job_tags.intersection(tags_to_avoid)
|
52
53
|
if has_tags_to_avoid:
|
53
54
|
if verbose:
|
54
|
-
|
55
|
+
log(
|
55
56
|
(
|
56
57
|
f"not running job '{job['label']}' because it contains the "
|
57
58
|
f"following tags: {printable_set(has_tags_to_avoid)}"
|
@@ -73,17 +74,23 @@ def filter_jobs(
|
|
73
74
|
jobs: PhaseGroupedJobs = config_metadata.jobs
|
74
75
|
verbose: bool = config_metadata.args.verbose
|
75
76
|
if tags_to_run:
|
76
|
-
|
77
|
+
log(f"filtering for tags {printable_set(tags_to_run)}", decorate=True, end="")
|
77
78
|
if tags_to_avoid:
|
78
79
|
if tags_to_run:
|
79
|
-
|
80
|
-
|
80
|
+
log(", ", decorate=False, end="")
|
81
|
+
else:
|
82
|
+
log(decorate=True, end="")
|
83
|
+
log(
|
84
|
+
f"excluding jobs with tags {printable_set(tags_to_avoid)}",
|
85
|
+
decorate=False,
|
86
|
+
end="",
|
87
|
+
)
|
81
88
|
if tags_to_run or tags_to_avoid:
|
82
|
-
|
89
|
+
log(decorate=False)
|
83
90
|
filtered_jobs: PhaseGroupedJobs = defaultdict(list)
|
84
91
|
for phase in config_metadata.phases:
|
85
92
|
if phase not in phases_to_run:
|
86
|
-
|
93
|
+
log(f"skipping phase '{phase}'")
|
87
94
|
continue
|
88
95
|
_get_jobs_matching(
|
89
96
|
phase=phase,
|
@@ -95,10 +102,10 @@ def filter_jobs(
|
|
95
102
|
verbose=verbose,
|
96
103
|
)
|
97
104
|
if len(filtered_jobs[phase]) == 0:
|
98
|
-
|
105
|
+
log(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
|
99
106
|
continue
|
100
107
|
|
101
|
-
|
102
|
-
|
108
|
+
log((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
|
109
|
+
log(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
|
103
110
|
|
104
111
|
return filtered_jobs
|
@@ -20,20 +20,35 @@ def _load_python_function_from_module(
|
|
20
20
|
).absolute()
|
21
21
|
|
22
22
|
# load the function
|
23
|
-
|
24
|
-
|
23
|
+
try:
|
24
|
+
module_spec = module_spec_from_file_location(
|
25
|
+
function_to_load, abs_module_file_path
|
26
|
+
)
|
27
|
+
if not module_spec:
|
28
|
+
raise FileNotFoundError()
|
29
|
+
if not module_spec.loader:
|
30
|
+
raise FunctionNotFound("unable to load module")
|
31
|
+
except FileNotFoundError as err:
|
25
32
|
raise FunctionNotFound(
|
26
33
|
(
|
27
34
|
f"unable to load '{function_to_load}' from '{str(module_file_path)} "
|
28
35
|
f"relative to '{str(cfg_filepath)}"
|
29
36
|
)
|
30
|
-
)
|
37
|
+
) from err
|
31
38
|
|
32
39
|
module = module_from_spec(module_spec)
|
33
|
-
|
34
|
-
if not module_spec.loader:
|
40
|
+
if not module:
|
35
41
|
raise FunctionNotFound("unable to load module")
|
36
|
-
|
42
|
+
sys.modules[module_name] = module
|
43
|
+
try:
|
44
|
+
module_spec.loader.exec_module(module)
|
45
|
+
except FileNotFoundError as err:
|
46
|
+
raise FunctionNotFound(
|
47
|
+
(
|
48
|
+
f"unable to load '{function_to_load}' from '{str(module_file_path)} "
|
49
|
+
f"relative to '{str(cfg_filepath)}"
|
50
|
+
)
|
51
|
+
) from err
|
37
52
|
try:
|
38
53
|
function: JobFunction = getattr(module, function_to_load)
|
39
54
|
except AttributeError as err:
|
@@ -70,7 +85,7 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
|
|
70
85
|
return module_path.relative_to(cfg_filepath.parent.absolute())
|
71
86
|
|
72
87
|
|
73
|
-
def
|
88
|
+
def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
|
74
89
|
"""Given a job-description dynamically loads the job-function so we can call it.
|
75
90
|
|
76
91
|
Side-effects: also re-addressed the job-config.
|
@@ -0,0 +1,11 @@
|
|
1
|
+
import typing
|
2
|
+
|
3
|
+
|
4
|
+
def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
|
5
|
+
"""Thin wrapper around 'print', so we can change the output.
|
6
|
+
|
7
|
+
One way we change it is to decorate the output with 'runem'
|
8
|
+
"""
|
9
|
+
if decorate:
|
10
|
+
msg = f"runem: {msg}"
|
11
|
+
print(msg, end=end)
|
@@ -2,6 +2,7 @@ import typing
|
|
2
2
|
from collections import defaultdict
|
3
3
|
from datetime import timedelta
|
4
4
|
|
5
|
+
from runem.log import log
|
5
6
|
from runem.types import (
|
6
7
|
JobReturn,
|
7
8
|
JobRunMetadatasByPhase,
|
@@ -25,12 +26,15 @@ def _plot_times(
|
|
25
26
|
phase_run_oder: OrderedPhases,
|
26
27
|
timing_data: JobRunTimesByPhase,
|
27
28
|
) -> timedelta:
|
28
|
-
"""Prints a report to terminal on how well we performed.
|
29
|
+
"""Prints a report to terminal on how well we performed.
|
30
|
+
|
31
|
+
Also calculates the wall-clock time-saved for the user.
|
32
|
+
"""
|
29
33
|
labels: typing.List[str] = []
|
30
34
|
times: typing.List[float] = []
|
31
35
|
job_time_sum: timedelta = timedelta() # init to 0
|
32
36
|
for phase in phase_run_oder:
|
33
|
-
#
|
37
|
+
# log(f"Phase '{phase}' jobs took:")
|
34
38
|
phase_total_time: float = 0.0
|
35
39
|
phase_start_idx = len(labels)
|
36
40
|
for label, job_time in timing_data[phase]:
|
@@ -59,18 +63,37 @@ def _plot_times(
|
|
59
63
|
fig.show()
|
60
64
|
else: # pragma: FIXME: add code coverage
|
61
65
|
for label, time in zip(labels, times):
|
62
|
-
|
66
|
+
log(f"{label}: {time}s")
|
63
67
|
|
64
68
|
time_saved: timedelta = job_time_sum - overall_run_time
|
65
69
|
return time_saved
|
66
70
|
|
67
71
|
|
72
|
+
def _print_reports_by_phase(
|
73
|
+
phase_run_oder: OrderedPhases, report_data: JobRunReportByPhase
|
74
|
+
) -> None:
|
75
|
+
"""Logs out the reports by grouped by phase."""
|
76
|
+
for phase in phase_run_oder:
|
77
|
+
report_urls: ReportUrls = report_data[phase]
|
78
|
+
job_report_url_info: ReportUrlInfo
|
79
|
+
for job_report_url_info in report_urls:
|
80
|
+
if not job_report_url_info:
|
81
|
+
continue
|
82
|
+
log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
|
83
|
+
|
84
|
+
|
68
85
|
def report_on_run(
|
69
86
|
phase_run_oder: OrderedPhases,
|
70
87
|
job_run_metadatas: JobRunMetadatasByPhase,
|
71
88
|
overall_runtime: timedelta,
|
72
|
-
):
|
73
|
-
|
89
|
+
) -> timedelta:
|
90
|
+
"""Generate high-level reports AND prints out any reports returned by jobs.
|
91
|
+
|
92
|
+
IMPORTANT: returns the wall-clock time saved to the user.
|
93
|
+
"""
|
94
|
+
log("reports:")
|
95
|
+
|
96
|
+
# First, collate all data, timing and reports
|
74
97
|
timing_data: JobRunTimesByPhase = defaultdict(list)
|
75
98
|
report_data: JobRunReportByPhase = defaultdict(list)
|
76
99
|
phase: PhaseName
|
@@ -80,19 +103,21 @@ def report_on_run(
|
|
80
103
|
for timing, reports in job_run_metadatas[phase]:
|
81
104
|
timing_data[phase].append(timing)
|
82
105
|
if reports:
|
106
|
+
# the job returned some report urls, record them against the
|
107
|
+
# job's phase
|
83
108
|
report_data[phase].extend(reports["reportUrls"])
|
109
|
+
|
110
|
+
# Now plot the times on the terminal to give a visual report of the timing.
|
111
|
+
# Also, calculate the time saved by runem, a key selling-point metric
|
84
112
|
time_saved: timedelta = _plot_times(
|
85
113
|
overall_run_time=overall_runtime,
|
86
114
|
phase_run_oder=phase_run_oder,
|
87
115
|
timing_data=timing_data,
|
88
116
|
)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
print(
|
96
|
-
f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}"
|
97
|
-
)
|
117
|
+
|
118
|
+
# Penultimate-ly print out the available reports grouped by run-phase.
|
119
|
+
_print_reports_by_phase(phase_run_oder, report_data)
|
120
|
+
|
121
|
+
# Return the key metric for runem, the wall-clock time saved to the user
|
122
|
+
# TODO: write this to disk
|
98
123
|
return time_saved
|
@@ -5,6 +5,8 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
|
|
5
5
|
from subprocess import CompletedProcess
|
6
6
|
from subprocess import run as subprocess_run
|
7
7
|
|
8
|
+
from runem.log import log
|
9
|
+
|
8
10
|
TERMINAL_WIDTH = 86
|
9
11
|
|
10
12
|
|
@@ -42,10 +44,10 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
42
44
|
"""Runs the given command, returning stdout or throwing on any error."""
|
43
45
|
cmd_string: str = " ".join(cmd)
|
44
46
|
if verbose:
|
45
|
-
|
47
|
+
log(f"running: start: {label}: {cmd_string}")
|
46
48
|
if valid_exit_ids is not None:
|
47
49
|
valid_exit_strs = ",".join([str(exit_code) for exit_code in valid_exit_ids])
|
48
|
-
|
50
|
+
log(f"\tallowed return ids are: {valid_exit_strs}")
|
49
51
|
|
50
52
|
if valid_exit_ids is None:
|
51
53
|
valid_exit_ids = (0,)
|
@@ -60,13 +62,13 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
60
62
|
run_env_as_string = " ".join(
|
61
63
|
[f"{key}='{value}'" for key, value in run_env.items()]
|
62
64
|
)
|
63
|
-
|
65
|
+
log(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
|
64
66
|
|
65
67
|
if env_overrides:
|
66
68
|
env_overrides_as_string = " ".join(
|
67
69
|
[f"{key}='{value}'" for key, value in env_overrides.items()]
|
68
70
|
)
|
69
|
-
|
71
|
+
log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
|
70
72
|
|
71
73
|
env_overrides_dict = {}
|
72
74
|
if env_overrides:
|
@@ -127,6 +129,6 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
|
|
127
129
|
assert process is not None
|
128
130
|
cmd_stdout: str = get_stdout(process, prefix=label)
|
129
131
|
if verbose:
|
130
|
-
|
131
|
-
|
132
|
+
log(cmd_stdout)
|
133
|
+
log(f"running: done: {label}: {cmd_string}")
|
132
134
|
return cmd_stdout
|