runem 0.0.16__tar.gz → 0.0.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. {runem-0.0.16 → runem-0.0.18}/HISTORY.md +114 -0
  2. {runem-0.0.16 → runem-0.0.18}/PKG-INFO +3 -2
  3. runem-0.0.18/runem/VERSION +1 -0
  4. runem-0.0.18/runem/cli/initialise_options.py +25 -0
  5. {runem-0.0.16 → runem-0.0.18}/runem/command_line.py +7 -6
  6. {runem-0.0.16 → runem-0.0.18}/runem/config.py +2 -1
  7. {runem-0.0.16 → runem-0.0.18}/runem/config_parse.py +6 -5
  8. runem-0.0.16/runem/job_runner.py → runem-0.0.18/runem/job_execute.py +17 -10
  9. {runem-0.0.16 → runem-0.0.18}/runem/job_filter.py +18 -11
  10. runem-0.0.16/runem/job_function_python.py → runem-0.0.18/runem/job_wrapper_python.py +22 -7
  11. runem-0.0.18/runem/log.py +11 -0
  12. {runem-0.0.16 → runem-0.0.18}/runem/report.py +39 -14
  13. {runem-0.0.16 → runem-0.0.18}/runem/run_command.py +8 -6
  14. {runem-0.0.16 → runem-0.0.18}/runem/runem.py +54 -17
  15. {runem-0.0.16 → runem-0.0.18}/runem/types.py +3 -2
  16. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/PKG-INFO +3 -2
  17. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/SOURCES.txt +7 -4
  18. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/requires.txt +2 -1
  19. runem-0.0.18/tests/cli/test_initialise_options.py +101 -0
  20. {runem-0.0.16 → runem-0.0.18}/tests/data/help_output.txt +13 -15
  21. {runem-0.0.16 → runem-0.0.18}/tests/test_config_parse.py +42 -0
  22. runem-0.0.16/tests/test_job_runner.py → runem-0.0.18/tests/test_job_execute.py +11 -11
  23. runem-0.0.16/tests/test_job_function_python.py → runem-0.0.18/tests/test_job_wrapper_python.py +86 -9
  24. {runem-0.0.16 → runem-0.0.18}/tests/test_report.py +41 -3
  25. {runem-0.0.16 → runem-0.0.18}/tests/test_run_command.py +8 -8
  26. {runem-0.0.16 → runem-0.0.18}/tests/test_runem.py +212 -120
  27. runem-0.0.16/runem/VERSION +0 -1
  28. {runem-0.0.16 → runem-0.0.18}/Containerfile +0 -0
  29. {runem-0.0.16 → runem-0.0.18}/LICENSE +0 -0
  30. {runem-0.0.16 → runem-0.0.18}/MANIFEST.in +0 -0
  31. {runem-0.0.16 → runem-0.0.18}/README.md +0 -0
  32. {runem-0.0.16 → runem-0.0.18}/runem/__init__.py +0 -0
  33. {runem-0.0.16 → runem-0.0.18}/runem/__main__.py +0 -0
  34. {runem-0.0.16 → runem-0.0.18}/runem/base.py +0 -0
  35. {runem-0.0.16 → runem-0.0.18}/runem/cli.py +0 -0
  36. {runem-0.0.16 → runem-0.0.18}/runem/config_metadata.py +0 -0
  37. {runem-0.0.16 → runem-0.0.18}/runem/files.py +0 -0
  38. {runem-0.0.16 → runem-0.0.18}/runem/py.typed +0 -0
  39. {runem-0.0.16 → runem-0.0.18}/runem/utils.py +0 -0
  40. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/dependency_links.txt +0 -0
  41. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/entry_points.txt +0 -0
  42. {runem-0.0.16 → runem-0.0.18}/runem.egg-info/top_level.txt +0 -0
  43. {runem-0.0.16 → runem-0.0.18}/setup.cfg +0 -0
  44. {runem-0.0.16 → runem-0.0.18}/setup.py +0 -0
  45. {runem-0.0.16 → runem-0.0.18}/tests/__init__.py +0 -0
  46. {runem-0.0.16 → runem-0.0.18}/tests/conftest.py +0 -0
  47. {runem-0.0.16 → runem-0.0.18}/tests/test_base.py +0 -0
  48. {runem-0.0.16 → runem-0.0.18}/tests/test_cli.py +0 -0
  49. {runem-0.0.16 → runem-0.0.18}/tests/test_config.py +0 -0
  50. {runem-0.0.16 → runem-0.0.18}/tests/test_files.py +0 -0
@@ -4,6 +4,120 @@ Changelog
4
4
 
5
5
  (unreleased)
6
6
  ------------
7
+ - Merge pull request #9 from lursight/chore/rename_core_functions.
8
+ [Frank Harrison]
9
+
10
+ Chore/rename core functions
11
+ - Chore(core-func-rename): renames functions and fixes imports. [Frank
12
+ Harrison]
13
+ - Chore(core-func-rename): renames files to better reflect contents.
14
+ [Frank Harrison]
15
+ - Merge pull request #8 from lursight/chore/more_typing_strictness.
16
+ [Frank Harrison]
17
+
18
+ Chore/more typing strictness
19
+ - Chore(mypy-strict): switches mypy to use strict-mode. [Frank Harrison]
20
+
21
+ This should catch more issues down the line
22
+ - Chore(mypy-strict): enable disallow_untyped_calls and annotate it.
23
+ [Frank Harrison]
24
+ - Chore(mypy-strict): enable disallow_untyped_defs in mypy. [Frank
25
+ Harrison]
26
+ - Chore(mypy-strict): enables check_untyped_defs in mypy. [Frank
27
+ Harrison]
28
+ - Chore(mypy-strict): annotates mypy config options. [Frank Harrison]
29
+ - Merge pull request #7 from lursight/chore/project_status. [Frank
30
+ Harrison]
31
+
32
+ chore(project-status): removes the project rename from actions
33
+ - Chore(project-status): removes the project rename from actions. [Frank
34
+ Harrison]
35
+ - Merge pull request #5 from lursight/feat/job_spinner. [Frank Harrison]
36
+
37
+ Feat/job spinner
38
+ - Feat(progress): gets a progress spinner working. [Frank Harrison]
39
+ - Feat(progress): adds way to track running jobs for multip-proc jobs.
40
+ [Frank Harrison]
41
+ - Merge pull request #3 from
42
+ lursight/dependabot/github_actions/actions/setup-python-5. [Frank
43
+ Harrison]
44
+
45
+ chore(deps): bump actions/setup-python from 4 to 5
46
+ - Chore(deps): bump actions/setup-python from 4 to 5. [dependabot[bot]]
47
+
48
+ Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5.
49
+ - [Release notes](https://github.com/actions/setup-python/releases)
50
+ - [Commits](https://github.com/actions/setup-python/compare/v4...v5)
51
+
52
+ ---
53
+ updated-dependencies:
54
+ - dependency-name: actions/setup-python
55
+ dependency-type: direct:production
56
+ update-type: version-update:semver-major
57
+ ...
58
+ - Merge pull request #6 from lursight/fix/help_tests. [Frank Harrison]
59
+
60
+ Fix/help tests
61
+ - Chore(fix-help-tests): fixing the help-text test in ci/cd. [Frank
62
+ Harrison]
63
+ - Chore(pytest): stops reporting of coverage on test failures. [Frank
64
+ Harrison]
65
+ - Chore(pytest): stops failing coverage BEFORE we want to. [Frank
66
+ Harrison]
67
+ - Chore(type): fixes type errors. [Frank Harrison]
68
+ - Merge pull request #4 from lursight/fix/python3.10_support. [Frank
69
+ Harrison]
70
+
71
+ fix(python3.10): removes another pipe shortcut
72
+ - Fix(python3.10): removes another pipe shortcut. [Frank Harrison]
73
+ - Merge branch 'fix/python3.10_support' [Frank Harrison]
74
+ - Fix(python3.10): removes newer typing syntax. [Frank Harrison]
75
+ - Merge branch 'fix/coverage' [Frank Harrison]
76
+ - Fix(coverage): adds more coverage to parse_config() [Frank Harrison]
77
+
78
+ ... specifically the bit that warns if we have to nordered phases
79
+ - Fix(coverage): adds more coverage to
80
+ _load_python_function_from_module() [Frank Harrison]
81
+ - Fix(coverage): adds more coverage to initialise_options() [Frank
82
+ Harrison]
83
+ - Fix(coverage): adds more coverage to report.py. [Frank Harrison]
84
+ - Fix(coverage): annotates a file that needs more coverage. [Frank
85
+ Harrison]
86
+ - Merge branch 'fix/spell_check' [Frank Harrison]
87
+ - Fix(spell-check): fixes the spell-checker by ignoring the history
88
+ file. [Frank Harrison]
89
+
90
+ ... which contains typos in the commit hitsory
91
+
92
+
93
+ 0.0.17 (2023-12-09)
94
+ -------------------
95
+ - Release: version 0.0.17 🚀 [Frank Harrison]
96
+ - Merge branch 'chore/run_in_check_mode_on_release' [Frank Harrison]
97
+ - Chore(release-checks): makes each command fail on 'make release'
98
+ [Frank Harrison]
99
+ - Chore(release-checks): run runem in check mode on 'make release'
100
+ [Frank Harrison]
101
+ - Merge branch 'fix/pyyaml_dep' [Frank Harrison]
102
+ - Fix(deps): adds the py-yaml dep to release requirements. [Frank
103
+ Harrison]
104
+ - Merge branch 'chore/log_output' [Frank Harrison]
105
+ - Chore(coverage): fixes up coverage, for now. [Frank Harrison]
106
+ - Fixup: logs. [Frank Harrison]
107
+ - Chore(black): formats log.py. [Frank Harrison]
108
+ - Chore(log-format): replaces print() with log. [Frank Harrison]
109
+
110
+ ... also adds a prefix to the logging
111
+ - Merge branch 'chore/skip_spell_check_history' [Frank Harrison]
112
+ - Chore(spell-history): add cSpell:disable to HISTORY.md's frontmatter.
113
+ [Frank Harrison]
114
+
115
+ ... because some of the commit messages contain spelling typos
116
+
117
+
118
+ 0.0.16 (2023-12-05)
119
+ -------------------
120
+ - Release: version 0.0.16 🚀 [Frank Harrison]
7
121
  - Merge branch 'chore/get_release_running_tests' [Frank Harrison]
8
122
  - Chore(test-on-release): prints existing tags on make release. [Frank
9
123
  Harrison]
@@ -1,11 +1,13 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: runem
3
- Version: 0.0.16
3
+ Version: 0.0.18
4
4
  Summary: Awesome runem created by lursight
5
5
  Home-page: https://github.com/lursight/runem/
6
6
  Author: lursight
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
+ Requires-Dist: halo
10
+ Requires-Dist: PyYAML
9
11
  Provides-Extra: test
10
12
  Requires-Dist: black==23.11.0; extra == "test"
11
13
  Requires-Dist: coverage==7.3.2; extra == "test"
@@ -23,7 +25,6 @@ Requires-Dist: pytest-cov==4.1.0; extra == "test"
23
25
  Requires-Dist: pytest-profiling==1.7.0; extra == "test"
24
26
  Requires-Dist: pytest-xdist==3.3.1; extra == "test"
25
27
  Requires-Dist: pytest==7.4.3; extra == "test"
26
- Requires-Dist: PyYAML==6.0.1; extra == "test"
27
28
  Requires-Dist: termplotlib==0.3.9; extra == "test"
28
29
  Requires-Dist: types-PyYAML==6.0.12.12; extra == "test"
29
30
  Requires-Dist: requests-mock==1.10.0; extra == "test"
@@ -0,0 +1 @@
1
+ 0.0.18
@@ -0,0 +1,25 @@
1
+ import argparse
2
+
3
+ from runem.config_metadata import ConfigMetadata
4
+ from runem.types import Options
5
+
6
+
7
+ def initialise_options(
8
+ config_metadata: ConfigMetadata,
9
+ args: argparse.Namespace,
10
+ ) -> Options:
11
+ """Initialises and returns the set of options to use for this run.
12
+
13
+ Returns the options dictionary
14
+ """
15
+
16
+ options: Options = {
17
+ option["name"]: option["default"] for option in config_metadata.options_config
18
+ }
19
+ if config_metadata.options_config and args.overrides_on: # pragma: no branch
20
+ for option_name in args.overrides_on: # pragma: no branch
21
+ options[option_name] = True
22
+ if config_metadata.options_config and args.overrides_off: # pragma: no branch
23
+ for option_name in args.overrides_off:
24
+ options[option_name] = False
25
+ return options
@@ -5,6 +5,7 @@ import sys
5
5
  import typing
6
6
 
7
7
  from runem.config_metadata import ConfigMetadata
8
+ from runem.log import log
8
9
  from runem.types import JobNames, OptionConfig, Options
9
10
  from runem.utils import printable_set
10
11
 
@@ -146,7 +147,7 @@ def parse_args(
146
147
 
147
148
  args = parser.parse_args(argv[1:])
148
149
 
149
- options: Options = _initialise_options(config_metadata, args)
150
+ options: Options = initialise_options(config_metadata, args)
150
151
 
151
152
  if not _validate_filters(config_metadata, args):
152
153
  sys.exit(1)
@@ -176,7 +177,7 @@ def _validate_filters(
176
177
  for name, name_list in (("--jobs", args.jobs), ("--not-jobs", args.jobs_excluded)):
177
178
  for job_name in name_list:
178
179
  if job_name not in config_metadata.all_job_names:
179
- print(
180
+ log(
180
181
  (
181
182
  f"ERROR: invalid job-name '{job_name}' for {name}, "
182
183
  f"choose from one of {printable_set(config_metadata.all_job_names)}"
@@ -188,7 +189,7 @@ def _validate_filters(
188
189
  for name, tag_list in (("--tags", args.tags), ("--not-tags", args.tags_excluded)):
189
190
  for tag in tag_list:
190
191
  if tag not in config_metadata.all_job_tags:
191
- print(
192
+ log(
192
193
  (
193
194
  f"ERROR: invalid tag '{tag}' for {name}, "
194
195
  f"choose from one of {printable_set(config_metadata.all_job_tags)}"
@@ -203,7 +204,7 @@ def _validate_filters(
203
204
  ):
204
205
  for phase in phase_list:
205
206
  if phase not in config_metadata.all_job_phases:
206
- print(
207
+ log(
207
208
  (
208
209
  f"ERROR: invalid phase '{phase}' for {name}, "
209
210
  f"choose from one of {printable_set(config_metadata.all_job_phases)}"
@@ -213,7 +214,7 @@ def _validate_filters(
213
214
  return True
214
215
 
215
216
 
216
- def _initialise_options(
217
+ def initialise_options(
217
218
  config_metadata: ConfigMetadata,
218
219
  args: argparse.Namespace,
219
220
  ) -> Options:
@@ -226,7 +227,7 @@ def _initialise_options(
226
227
  option["name"]: option["default"] for option in config_metadata.options_config
227
228
  }
228
229
  if config_metadata.options_config and args.overrides_on: # pragma: no branch
229
- for option_name in args.overrides_on:
230
+ for option_name in args.overrides_on: # pragma: no branch
230
231
  options[option_name] = True
231
232
  if config_metadata.options_config and args.overrides_off: # pragma: no branch
232
233
  for option_name in args.overrides_off:
@@ -4,6 +4,7 @@ import typing
4
4
 
5
5
  import yaml
6
6
 
7
+ from runem.log import log
7
8
  from runem.types import Config
8
9
 
9
10
  CFG_FILE_YAML = pathlib.Path(".runem.yml")
@@ -47,7 +48,7 @@ def _find_cfg() -> pathlib.Path:
47
48
  return cfg_candidate
48
49
 
49
50
  # error out and exit as we currently require the cfg file as it lists jobs.
50
- print(f"ERROR: Config not found! Looked from {start_dirs}")
51
+ log(f"ERROR: Config not found! Looked from {start_dirs}")
51
52
  sys.exit(1)
52
53
 
53
54
 
@@ -4,7 +4,8 @@ import typing
4
4
  from collections import defaultdict
5
5
 
6
6
  from runem.config_metadata import ConfigMetadata
7
- from runem.job_function_python import get_job_function
7
+ from runem.job_wrapper_python import get_job_wrapper
8
+ from runem.log import log
8
9
  from runem.types import (
9
10
  Config,
10
11
  ConfigNodes,
@@ -73,12 +74,12 @@ def parse_job_config(
73
74
  try:
74
75
  job_names_used = job["label"] in in_out_job_names
75
76
  if job_names_used:
76
- print("ERROR: duplicate job label!")
77
- print(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
77
+ log("ERROR: duplicate job label!")
78
+ log(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
78
79
  sys.exit(1)
79
80
 
80
81
  # try and load the function _before_ we schedule it's execution
81
- get_job_function(job, cfg_filepath)
82
+ get_job_wrapper(job, cfg_filepath)
82
83
  phase_id: PhaseName = job["when"]["phase"]
83
84
  in_out_jobs_by_phase[phase_id].append(job)
84
85
 
@@ -138,7 +139,7 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
138
139
  )
139
140
 
140
141
  if not phase_order:
141
- print("WARNING: phase ordering not configured! Runs will be non-deterministic!")
142
+ log("WARNING: phase ordering not configured! Runs will be non-deterministic!")
142
143
  phase_order = tuple(job_phases)
143
144
 
144
145
  # tags = tags.union(("python", "es", "firebase_funcs"))
@@ -2,15 +2,17 @@ import inspect
2
2
  import os
3
3
  import pathlib
4
4
  import typing
5
+ import uuid
5
6
  from datetime import timedelta
6
7
  from timeit import default_timer as timer
7
8
 
8
9
  from runem.config_metadata import ConfigMetadata
9
- from runem.job_function_python import get_job_function
10
+ from runem.job_wrapper_python import get_job_wrapper
11
+ from runem.log import log
10
12
  from runem.types import FilePathList, FilePathListLookup, JobConfig, JobReturn, JobTags
11
13
 
12
14
 
13
- def job_runner_inner(
15
+ def job_execute_inner(
14
16
  job_config: JobConfig,
15
17
  config_metadata: ConfigMetadata,
16
18
  file_lists: FilePathListLookup,
@@ -21,12 +23,12 @@ def job_runner_inner(
21
23
  """
22
24
  label = job_config["label"]
23
25
  if config_metadata.args.verbose:
24
- print(f"START: {label}")
26
+ log(f"START: {label}")
25
27
  root_path: pathlib.Path = config_metadata.cfg_filepath.parent
26
28
  function: typing.Callable
27
29
  job_tags: JobTags = set(job_config["when"]["tags"])
28
30
  os.chdir(root_path)
29
- function = get_job_function(job_config, config_metadata.cfg_filepath)
31
+ function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
30
32
 
31
33
  # get the files for all files found for this job's tags
32
34
  file_list: FilePathList = []
@@ -36,7 +38,7 @@ def job_runner_inner(
36
38
 
37
39
  if not file_list:
38
40
  # no files to work on
39
- print(f"WARNING: skipping job '{label}', no files for job")
41
+ log(f"WARNING: skipping job '{label}', no files for job")
40
42
  return (f"{label}: no files!", timedelta(0)), None
41
43
 
42
44
  if (
@@ -52,7 +54,7 @@ def job_runner_inner(
52
54
  start = timer()
53
55
  func_signature = inspect.signature(function)
54
56
  if config_metadata.args.verbose:
55
- print(f"job: running {job_config['label']}")
57
+ log(f"job: running {job_config['label']}")
56
58
  reports: JobReturn
57
59
  if "args" in func_signature.parameters:
58
60
  reports = function(config_metadata.args, config_metadata.options, file_list)
@@ -68,18 +70,23 @@ def job_runner_inner(
68
70
  end = timer()
69
71
  time_taken: timedelta = timedelta(seconds=end - start)
70
72
  if config_metadata.args.verbose:
71
- print(f"DONE: {label}: {time_taken}")
73
+ log(f"DONE: {label}: {time_taken}")
72
74
  timing_data = (label, time_taken)
73
75
  return (timing_data, reports)
74
76
 
75
77
 
76
- def job_runner(
78
+ def job_execute(
77
79
  job_config: JobConfig,
80
+ running_jobs: typing.Dict[str, str],
78
81
  config_metadata: ConfigMetadata,
79
82
  file_lists: FilePathListLookup,
80
83
  ) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
81
- """Thing wrapper around job_runner_inner needed fro mocking in tests.
84
+ """Thin-wrapper around job_execute_inner needed for mocking in tests.
82
85
 
83
86
  Needed for faster tests.
84
87
  """
85
- return job_runner_inner(job_config, config_metadata, file_lists)
88
+ this_id: str = str(uuid.uuid4())
89
+ running_jobs[this_id] = job_config["label"]
90
+ results = job_execute_inner(job_config, config_metadata, file_lists)
91
+ del running_jobs[this_id]
92
+ return results
@@ -2,6 +2,7 @@ import typing
2
2
  from collections import defaultdict
3
3
 
4
4
  from runem.config_metadata import ConfigMetadata
5
+ from runem.log import log
5
6
  from runem.types import (
6
7
  JobConfig,
7
8
  JobNames,
@@ -30,7 +31,7 @@ def _get_jobs_matching(
30
31
  matching_tags = job_tags.intersection(tags)
31
32
  if not matching_tags:
32
33
  if verbose:
33
- print(
34
+ log(
34
35
  (
35
36
  f"not running job '{job['label']}' because it doesn't have "
36
37
  f"any of the following tags: {printable_set(tags)}"
@@ -40,7 +41,7 @@ def _get_jobs_matching(
40
41
 
41
42
  if job["label"] not in job_names:
42
43
  if verbose:
43
- print(
44
+ log(
44
45
  (
45
46
  f"not running job '{job['label']}' because it isn't in the "
46
47
  f"list of job names. See --jobs and --not-jobs"
@@ -51,7 +52,7 @@ def _get_jobs_matching(
51
52
  has_tags_to_avoid = job_tags.intersection(tags_to_avoid)
52
53
  if has_tags_to_avoid:
53
54
  if verbose:
54
- print(
55
+ log(
55
56
  (
56
57
  f"not running job '{job['label']}' because it contains the "
57
58
  f"following tags: {printable_set(has_tags_to_avoid)}"
@@ -73,17 +74,23 @@ def filter_jobs(
73
74
  jobs: PhaseGroupedJobs = config_metadata.jobs
74
75
  verbose: bool = config_metadata.args.verbose
75
76
  if tags_to_run:
76
- print(f"filtering for tags {printable_set(tags_to_run)}", end="")
77
+ log(f"filtering for tags {printable_set(tags_to_run)}", decorate=True, end="")
77
78
  if tags_to_avoid:
78
79
  if tags_to_run:
79
- print(", ", end="")
80
- print(f"excluding jobs with tags {printable_set(tags_to_avoid)}", end="")
80
+ log(", ", decorate=False, end="")
81
+ else:
82
+ log(decorate=True, end="")
83
+ log(
84
+ f"excluding jobs with tags {printable_set(tags_to_avoid)}",
85
+ decorate=False,
86
+ end="",
87
+ )
81
88
  if tags_to_run or tags_to_avoid:
82
- print()
89
+ log(decorate=False)
83
90
  filtered_jobs: PhaseGroupedJobs = defaultdict(list)
84
91
  for phase in config_metadata.phases:
85
92
  if phase not in phases_to_run:
86
- print(f"skipping phase '{phase}'")
93
+ log(f"skipping phase '{phase}'")
87
94
  continue
88
95
  _get_jobs_matching(
89
96
  phase=phase,
@@ -95,10 +102,10 @@ def filter_jobs(
95
102
  verbose=verbose,
96
103
  )
97
104
  if len(filtered_jobs[phase]) == 0:
98
- print(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
105
+ log(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
99
106
  continue
100
107
 
101
- print((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
102
- print(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
108
+ log((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
109
+ log(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
103
110
 
104
111
  return filtered_jobs
@@ -20,20 +20,35 @@ def _load_python_function_from_module(
20
20
  ).absolute()
21
21
 
22
22
  # load the function
23
- module_spec = module_spec_from_file_location(function_to_load, abs_module_file_path)
24
- if not module_spec:
23
+ try:
24
+ module_spec = module_spec_from_file_location(
25
+ function_to_load, abs_module_file_path
26
+ )
27
+ if not module_spec:
28
+ raise FileNotFoundError()
29
+ if not module_spec.loader:
30
+ raise FunctionNotFound("unable to load module")
31
+ except FileNotFoundError as err:
25
32
  raise FunctionNotFound(
26
33
  (
27
34
  f"unable to load '{function_to_load}' from '{str(module_file_path)} "
28
35
  f"relative to '{str(cfg_filepath)}"
29
36
  )
30
- )
37
+ ) from err
31
38
 
32
39
  module = module_from_spec(module_spec)
33
- sys.modules[module_name] = module
34
- if not module_spec.loader:
40
+ if not module:
35
41
  raise FunctionNotFound("unable to load module")
36
- module_spec.loader.exec_module(module)
42
+ sys.modules[module_name] = module
43
+ try:
44
+ module_spec.loader.exec_module(module)
45
+ except FileNotFoundError as err:
46
+ raise FunctionNotFound(
47
+ (
48
+ f"unable to load '{function_to_load}' from '{str(module_file_path)} "
49
+ f"relative to '{str(cfg_filepath)}"
50
+ )
51
+ ) from err
37
52
  try:
38
53
  function: JobFunction = getattr(module, function_to_load)
39
54
  except AttributeError as err:
@@ -70,7 +85,7 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
70
85
  return module_path.relative_to(cfg_filepath.parent.absolute())
71
86
 
72
87
 
73
- def get_job_function(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
88
+ def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
74
89
  """Given a job-description dynamically loads the job-function so we can call it.
75
90
 
76
91
  Side-effects: also re-addressed the job-config.
@@ -0,0 +1,11 @@
1
+ import typing
2
+
3
+
4
+ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
5
+ """Thin wrapper around 'print', so we can change the output.
6
+
7
+ One way we change it is to decorate the output with 'runem'
8
+ """
9
+ if decorate:
10
+ msg = f"runem: {msg}"
11
+ print(msg, end=end)
@@ -2,6 +2,7 @@ import typing
2
2
  from collections import defaultdict
3
3
  from datetime import timedelta
4
4
 
5
+ from runem.log import log
5
6
  from runem.types import (
6
7
  JobReturn,
7
8
  JobRunMetadatasByPhase,
@@ -25,12 +26,15 @@ def _plot_times(
25
26
  phase_run_oder: OrderedPhases,
26
27
  timing_data: JobRunTimesByPhase,
27
28
  ) -> timedelta:
28
- """Prints a report to terminal on how well we performed."""
29
+ """Prints a report to terminal on how well we performed.
30
+
31
+ Also calculates the wall-clock time-saved for the user.
32
+ """
29
33
  labels: typing.List[str] = []
30
34
  times: typing.List[float] = []
31
35
  job_time_sum: timedelta = timedelta() # init to 0
32
36
  for phase in phase_run_oder:
33
- # print(f"Phase '{phase}' jobs took:")
37
+ # log(f"Phase '{phase}' jobs took:")
34
38
  phase_total_time: float = 0.0
35
39
  phase_start_idx = len(labels)
36
40
  for label, job_time in timing_data[phase]:
@@ -59,18 +63,37 @@ def _plot_times(
59
63
  fig.show()
60
64
  else: # pragma: FIXME: add code coverage
61
65
  for label, time in zip(labels, times):
62
- print(f"{label}: {time}s")
66
+ log(f"{label}: {time}s")
63
67
 
64
68
  time_saved: timedelta = job_time_sum - overall_run_time
65
69
  return time_saved
66
70
 
67
71
 
72
+ def _print_reports_by_phase(
73
+ phase_run_oder: OrderedPhases, report_data: JobRunReportByPhase
74
+ ) -> None:
75
+ """Logs out the reports by grouped by phase."""
76
+ for phase in phase_run_oder:
77
+ report_urls: ReportUrls = report_data[phase]
78
+ job_report_url_info: ReportUrlInfo
79
+ for job_report_url_info in report_urls:
80
+ if not job_report_url_info:
81
+ continue
82
+ log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
83
+
84
+
68
85
  def report_on_run(
69
86
  phase_run_oder: OrderedPhases,
70
87
  job_run_metadatas: JobRunMetadatasByPhase,
71
88
  overall_runtime: timedelta,
72
- ):
73
- print("runem: reports:")
89
+ ) -> timedelta:
90
+ """Generate high-level reports AND prints out any reports returned by jobs.
91
+
92
+ IMPORTANT: returns the wall-clock time saved to the user.
93
+ """
94
+ log("reports:")
95
+
96
+ # First, collate all data, timing and reports
74
97
  timing_data: JobRunTimesByPhase = defaultdict(list)
75
98
  report_data: JobRunReportByPhase = defaultdict(list)
76
99
  phase: PhaseName
@@ -80,19 +103,21 @@ def report_on_run(
80
103
  for timing, reports in job_run_metadatas[phase]:
81
104
  timing_data[phase].append(timing)
82
105
  if reports:
106
+ # the job returned some report urls, record them against the
107
+ # job's phase
83
108
  report_data[phase].extend(reports["reportUrls"])
109
+
110
+ # Now plot the times on the terminal to give a visual report of the timing.
111
+ # Also, calculate the time saved by runem, a key selling-point metric
84
112
  time_saved: timedelta = _plot_times(
85
113
  overall_run_time=overall_runtime,
86
114
  phase_run_oder=phase_run_oder,
87
115
  timing_data=timing_data,
88
116
  )
89
- for phase in phase_run_oder:
90
- report_urls: ReportUrls = report_data[phase]
91
- job_report_url_info: ReportUrlInfo
92
- for job_report_url_info in report_urls:
93
- if not job_report_url_info:
94
- continue
95
- print(
96
- f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}"
97
- )
117
+
118
+ # Penultimate-ly print out the available reports grouped by run-phase.
119
+ _print_reports_by_phase(phase_run_oder, report_data)
120
+
121
+ # Return the key metric for runem, the wall-clock time saved to the user
122
+ # TODO: write this to disk
98
123
  return time_saved
@@ -5,6 +5,8 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
5
5
  from subprocess import CompletedProcess
6
6
  from subprocess import run as subprocess_run
7
7
 
8
+ from runem.log import log
9
+
8
10
  TERMINAL_WIDTH = 86
9
11
 
10
12
 
@@ -42,10 +44,10 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
42
44
  """Runs the given command, returning stdout or throwing on any error."""
43
45
  cmd_string: str = " ".join(cmd)
44
46
  if verbose:
45
- print(f"runem: running: start: {label}: {cmd_string}")
47
+ log(f"running: start: {label}: {cmd_string}")
46
48
  if valid_exit_ids is not None:
47
49
  valid_exit_strs = ",".join([str(exit_code) for exit_code in valid_exit_ids])
48
- print(f"runem:\tallowed return ids are: {valid_exit_strs}")
50
+ log(f"\tallowed return ids are: {valid_exit_strs}")
49
51
 
50
52
  if valid_exit_ids is None:
51
53
  valid_exit_ids = (0,)
@@ -60,13 +62,13 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
60
62
  run_env_as_string = " ".join(
61
63
  [f"{key}='{value}'" for key, value in run_env.items()]
62
64
  )
63
- print(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
65
+ log(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
64
66
 
65
67
  if env_overrides:
66
68
  env_overrides_as_string = " ".join(
67
69
  [f"{key}='{value}'" for key, value in env_overrides.items()]
68
70
  )
69
- print(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
71
+ log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
70
72
 
71
73
  env_overrides_dict = {}
72
74
  if env_overrides:
@@ -127,6 +129,6 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
127
129
  assert process is not None
128
130
  cmd_stdout: str = get_stdout(process, prefix=label)
129
131
  if verbose:
130
- print(cmd_stdout)
131
- print(f"runem: running: done: {label}: {cmd_string}")
132
+ log(cmd_stdout)
133
+ log(f"running: done: {label}: {cmd_string}")
132
134
  return cmd_stdout