runem 0.0.16__py3-none-any.whl → 0.0.18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
runem/command_line.py CHANGED
@@ -5,6 +5,7 @@ import sys
5
5
  import typing
6
6
 
7
7
  from runem.config_metadata import ConfigMetadata
8
+ from runem.log import log
8
9
  from runem.types import JobNames, OptionConfig, Options
9
10
  from runem.utils import printable_set
10
11
 
@@ -146,7 +147,7 @@ def parse_args(
146
147
 
147
148
  args = parser.parse_args(argv[1:])
148
149
 
149
- options: Options = _initialise_options(config_metadata, args)
150
+ options: Options = initialise_options(config_metadata, args)
150
151
 
151
152
  if not _validate_filters(config_metadata, args):
152
153
  sys.exit(1)
@@ -176,7 +177,7 @@ def _validate_filters(
176
177
  for name, name_list in (("--jobs", args.jobs), ("--not-jobs", args.jobs_excluded)):
177
178
  for job_name in name_list:
178
179
  if job_name not in config_metadata.all_job_names:
179
- print(
180
+ log(
180
181
  (
181
182
  f"ERROR: invalid job-name '{job_name}' for {name}, "
182
183
  f"choose from one of {printable_set(config_metadata.all_job_names)}"
@@ -188,7 +189,7 @@ def _validate_filters(
188
189
  for name, tag_list in (("--tags", args.tags), ("--not-tags", args.tags_excluded)):
189
190
  for tag in tag_list:
190
191
  if tag not in config_metadata.all_job_tags:
191
- print(
192
+ log(
192
193
  (
193
194
  f"ERROR: invalid tag '{tag}' for {name}, "
194
195
  f"choose from one of {printable_set(config_metadata.all_job_tags)}"
@@ -203,7 +204,7 @@ def _validate_filters(
203
204
  ):
204
205
  for phase in phase_list:
205
206
  if phase not in config_metadata.all_job_phases:
206
- print(
207
+ log(
207
208
  (
208
209
  f"ERROR: invalid phase '{phase}' for {name}, "
209
210
  f"choose from one of {printable_set(config_metadata.all_job_phases)}"
@@ -213,7 +214,7 @@ def _validate_filters(
213
214
  return True
214
215
 
215
216
 
216
- def _initialise_options(
217
+ def initialise_options(
217
218
  config_metadata: ConfigMetadata,
218
219
  args: argparse.Namespace,
219
220
  ) -> Options:
@@ -226,7 +227,7 @@ def _initialise_options(
226
227
  option["name"]: option["default"] for option in config_metadata.options_config
227
228
  }
228
229
  if config_metadata.options_config and args.overrides_on: # pragma: no branch
229
- for option_name in args.overrides_on:
230
+ for option_name in args.overrides_on: # pragma: no branch
230
231
  options[option_name] = True
231
232
  if config_metadata.options_config and args.overrides_off: # pragma: no branch
232
233
  for option_name in args.overrides_off:
runem/config.py CHANGED
@@ -4,6 +4,7 @@ import typing
4
4
 
5
5
  import yaml
6
6
 
7
+ from runem.log import log
7
8
  from runem.types import Config
8
9
 
9
10
  CFG_FILE_YAML = pathlib.Path(".runem.yml")
@@ -47,7 +48,7 @@ def _find_cfg() -> pathlib.Path:
47
48
  return cfg_candidate
48
49
 
49
50
  # error out and exit as we currently require the cfg file as it lists jobs.
50
- print(f"ERROR: Config not found! Looked from {start_dirs}")
51
+ log(f"ERROR: Config not found! Looked from {start_dirs}")
51
52
  sys.exit(1)
52
53
 
53
54
 
runem/config_parse.py CHANGED
@@ -4,7 +4,8 @@ import typing
4
4
  from collections import defaultdict
5
5
 
6
6
  from runem.config_metadata import ConfigMetadata
7
- from runem.job_function_python import get_job_function
7
+ from runem.job_wrapper_python import get_job_wrapper
8
+ from runem.log import log
8
9
  from runem.types import (
9
10
  Config,
10
11
  ConfigNodes,
@@ -73,12 +74,12 @@ def parse_job_config(
73
74
  try:
74
75
  job_names_used = job["label"] in in_out_job_names
75
76
  if job_names_used:
76
- print("ERROR: duplicate job label!")
77
- print(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
77
+ log("ERROR: duplicate job label!")
78
+ log(f"\t'{job['label']}' is used twice or more in {str(cfg_filepath)}")
78
79
  sys.exit(1)
79
80
 
80
81
  # try and load the function _before_ we schedule it's execution
81
- get_job_function(job, cfg_filepath)
82
+ get_job_wrapper(job, cfg_filepath)
82
83
  phase_id: PhaseName = job["when"]["phase"]
83
84
  in_out_jobs_by_phase[phase_id].append(job)
84
85
 
@@ -138,7 +139,7 @@ def parse_config(config: Config, cfg_filepath: pathlib.Path) -> ConfigMetadata:
138
139
  )
139
140
 
140
141
  if not phase_order:
141
- print("WARNING: phase ordering not configured! Runs will be non-deterministic!")
142
+ log("WARNING: phase ordering not configured! Runs will be non-deterministic!")
142
143
  phase_order = tuple(job_phases)
143
144
 
144
145
  # tags = tags.union(("python", "es", "firebase_funcs"))
@@ -2,15 +2,17 @@ import inspect
2
2
  import os
3
3
  import pathlib
4
4
  import typing
5
+ import uuid
5
6
  from datetime import timedelta
6
7
  from timeit import default_timer as timer
7
8
 
8
9
  from runem.config_metadata import ConfigMetadata
9
- from runem.job_function_python import get_job_function
10
+ from runem.job_wrapper_python import get_job_wrapper
11
+ from runem.log import log
10
12
  from runem.types import FilePathList, FilePathListLookup, JobConfig, JobReturn, JobTags
11
13
 
12
14
 
13
- def job_runner_inner(
15
+ def job_execute_inner(
14
16
  job_config: JobConfig,
15
17
  config_metadata: ConfigMetadata,
16
18
  file_lists: FilePathListLookup,
@@ -21,12 +23,12 @@ def job_runner_inner(
21
23
  """
22
24
  label = job_config["label"]
23
25
  if config_metadata.args.verbose:
24
- print(f"START: {label}")
26
+ log(f"START: {label}")
25
27
  root_path: pathlib.Path = config_metadata.cfg_filepath.parent
26
28
  function: typing.Callable
27
29
  job_tags: JobTags = set(job_config["when"]["tags"])
28
30
  os.chdir(root_path)
29
- function = get_job_function(job_config, config_metadata.cfg_filepath)
31
+ function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
30
32
 
31
33
  # get the files for all files found for this job's tags
32
34
  file_list: FilePathList = []
@@ -36,7 +38,7 @@ def job_runner_inner(
36
38
 
37
39
  if not file_list:
38
40
  # no files to work on
39
- print(f"WARNING: skipping job '{label}', no files for job")
41
+ log(f"WARNING: skipping job '{label}', no files for job")
40
42
  return (f"{label}: no files!", timedelta(0)), None
41
43
 
42
44
  if (
@@ -52,7 +54,7 @@ def job_runner_inner(
52
54
  start = timer()
53
55
  func_signature = inspect.signature(function)
54
56
  if config_metadata.args.verbose:
55
- print(f"job: running {job_config['label']}")
57
+ log(f"job: running {job_config['label']}")
56
58
  reports: JobReturn
57
59
  if "args" in func_signature.parameters:
58
60
  reports = function(config_metadata.args, config_metadata.options, file_list)
@@ -68,18 +70,23 @@ def job_runner_inner(
68
70
  end = timer()
69
71
  time_taken: timedelta = timedelta(seconds=end - start)
70
72
  if config_metadata.args.verbose:
71
- print(f"DONE: {label}: {time_taken}")
73
+ log(f"DONE: {label}: {time_taken}")
72
74
  timing_data = (label, time_taken)
73
75
  return (timing_data, reports)
74
76
 
75
77
 
76
- def job_runner(
78
+ def job_execute(
77
79
  job_config: JobConfig,
80
+ running_jobs: typing.Dict[str, str],
78
81
  config_metadata: ConfigMetadata,
79
82
  file_lists: FilePathListLookup,
80
83
  ) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
81
- """Thing wrapper around job_runner_inner needed fro mocking in tests.
84
+ """Thin-wrapper around job_execute_inner needed for mocking in tests.
82
85
 
83
86
  Needed for faster tests.
84
87
  """
85
- return job_runner_inner(job_config, config_metadata, file_lists)
88
+ this_id: str = str(uuid.uuid4())
89
+ running_jobs[this_id] = job_config["label"]
90
+ results = job_execute_inner(job_config, config_metadata, file_lists)
91
+ del running_jobs[this_id]
92
+ return results
runem/job_filter.py CHANGED
@@ -2,6 +2,7 @@ import typing
2
2
  from collections import defaultdict
3
3
 
4
4
  from runem.config_metadata import ConfigMetadata
5
+ from runem.log import log
5
6
  from runem.types import (
6
7
  JobConfig,
7
8
  JobNames,
@@ -30,7 +31,7 @@ def _get_jobs_matching(
30
31
  matching_tags = job_tags.intersection(tags)
31
32
  if not matching_tags:
32
33
  if verbose:
33
- print(
34
+ log(
34
35
  (
35
36
  f"not running job '{job['label']}' because it doesn't have "
36
37
  f"any of the following tags: {printable_set(tags)}"
@@ -40,7 +41,7 @@ def _get_jobs_matching(
40
41
 
41
42
  if job["label"] not in job_names:
42
43
  if verbose:
43
- print(
44
+ log(
44
45
  (
45
46
  f"not running job '{job['label']}' because it isn't in the "
46
47
  f"list of job names. See --jobs and --not-jobs"
@@ -51,7 +52,7 @@ def _get_jobs_matching(
51
52
  has_tags_to_avoid = job_tags.intersection(tags_to_avoid)
52
53
  if has_tags_to_avoid:
53
54
  if verbose:
54
- print(
55
+ log(
55
56
  (
56
57
  f"not running job '{job['label']}' because it contains the "
57
58
  f"following tags: {printable_set(has_tags_to_avoid)}"
@@ -73,17 +74,23 @@ def filter_jobs(
73
74
  jobs: PhaseGroupedJobs = config_metadata.jobs
74
75
  verbose: bool = config_metadata.args.verbose
75
76
  if tags_to_run:
76
- print(f"filtering for tags {printable_set(tags_to_run)}", end="")
77
+ log(f"filtering for tags {printable_set(tags_to_run)}", decorate=True, end="")
77
78
  if tags_to_avoid:
78
79
  if tags_to_run:
79
- print(", ", end="")
80
- print(f"excluding jobs with tags {printable_set(tags_to_avoid)}", end="")
80
+ log(", ", decorate=False, end="")
81
+ else:
82
+ log(decorate=True, end="")
83
+ log(
84
+ f"excluding jobs with tags {printable_set(tags_to_avoid)}",
85
+ decorate=False,
86
+ end="",
87
+ )
81
88
  if tags_to_run or tags_to_avoid:
82
- print()
89
+ log(decorate=False)
83
90
  filtered_jobs: PhaseGroupedJobs = defaultdict(list)
84
91
  for phase in config_metadata.phases:
85
92
  if phase not in phases_to_run:
86
- print(f"skipping phase '{phase}'")
93
+ log(f"skipping phase '{phase}'")
87
94
  continue
88
95
  _get_jobs_matching(
89
96
  phase=phase,
@@ -95,10 +102,10 @@ def filter_jobs(
95
102
  verbose=verbose,
96
103
  )
97
104
  if len(filtered_jobs[phase]) == 0:
98
- print(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
105
+ log(f"No jobs for phase '{phase}' tags {printable_set(tags_to_run)}")
99
106
  continue
100
107
 
101
- print((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
102
- print(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
108
+ log((f"will run {len(filtered_jobs[phase])} jobs for phase '{phase}'"))
109
+ log(f"\t{[job['label'] for job in filtered_jobs[phase]]}")
103
110
 
104
111
  return filtered_jobs
@@ -20,20 +20,35 @@ def _load_python_function_from_module(
20
20
  ).absolute()
21
21
 
22
22
  # load the function
23
- module_spec = module_spec_from_file_location(function_to_load, abs_module_file_path)
24
- if not module_spec:
23
+ try:
24
+ module_spec = module_spec_from_file_location(
25
+ function_to_load, abs_module_file_path
26
+ )
27
+ if not module_spec:
28
+ raise FileNotFoundError()
29
+ if not module_spec.loader:
30
+ raise FunctionNotFound("unable to load module")
31
+ except FileNotFoundError as err:
25
32
  raise FunctionNotFound(
26
33
  (
27
34
  f"unable to load '{function_to_load}' from '{str(module_file_path)} "
28
35
  f"relative to '{str(cfg_filepath)}"
29
36
  )
30
- )
37
+ ) from err
31
38
 
32
39
  module = module_from_spec(module_spec)
33
- sys.modules[module_name] = module
34
- if not module_spec.loader:
40
+ if not module:
35
41
  raise FunctionNotFound("unable to load module")
36
- module_spec.loader.exec_module(module)
42
+ sys.modules[module_name] = module
43
+ try:
44
+ module_spec.loader.exec_module(module)
45
+ except FileNotFoundError as err:
46
+ raise FunctionNotFound(
47
+ (
48
+ f"unable to load '{function_to_load}' from '{str(module_file_path)} "
49
+ f"relative to '{str(cfg_filepath)}"
50
+ )
51
+ ) from err
37
52
  try:
38
53
  function: JobFunction = getattr(module, function_to_load)
39
54
  except AttributeError as err:
@@ -70,7 +85,7 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
70
85
  return module_path.relative_to(cfg_filepath.parent.absolute())
71
86
 
72
87
 
73
- def get_job_function(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
88
+ def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
74
89
  """Given a job-description dynamically loads the job-function so we can call it.
75
90
 
76
91
  Side-effects: also re-addressed the job-config.
runem/log.py ADDED
@@ -0,0 +1,11 @@
1
+ import typing
2
+
3
+
4
+ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
5
+ """Thin wrapper around 'print', so we can change the output.
6
+
7
+ One way we change it is to decorate the output with 'runem'
8
+ """
9
+ if decorate:
10
+ msg = f"runem: {msg}"
11
+ print(msg, end=end)
runem/report.py CHANGED
@@ -2,6 +2,7 @@ import typing
2
2
  from collections import defaultdict
3
3
  from datetime import timedelta
4
4
 
5
+ from runem.log import log
5
6
  from runem.types import (
6
7
  JobReturn,
7
8
  JobRunMetadatasByPhase,
@@ -25,12 +26,15 @@ def _plot_times(
25
26
  phase_run_oder: OrderedPhases,
26
27
  timing_data: JobRunTimesByPhase,
27
28
  ) -> timedelta:
28
- """Prints a report to terminal on how well we performed."""
29
+ """Prints a report to terminal on how well we performed.
30
+
31
+ Also calculates the wall-clock time-saved for the user.
32
+ """
29
33
  labels: typing.List[str] = []
30
34
  times: typing.List[float] = []
31
35
  job_time_sum: timedelta = timedelta() # init to 0
32
36
  for phase in phase_run_oder:
33
- # print(f"Phase '{phase}' jobs took:")
37
+ # log(f"Phase '{phase}' jobs took:")
34
38
  phase_total_time: float = 0.0
35
39
  phase_start_idx = len(labels)
36
40
  for label, job_time in timing_data[phase]:
@@ -59,18 +63,37 @@ def _plot_times(
59
63
  fig.show()
60
64
  else: # pragma: FIXME: add code coverage
61
65
  for label, time in zip(labels, times):
62
- print(f"{label}: {time}s")
66
+ log(f"{label}: {time}s")
63
67
 
64
68
  time_saved: timedelta = job_time_sum - overall_run_time
65
69
  return time_saved
66
70
 
67
71
 
72
+ def _print_reports_by_phase(
73
+ phase_run_oder: OrderedPhases, report_data: JobRunReportByPhase
74
+ ) -> None:
75
+ """Logs out the reports by grouped by phase."""
76
+ for phase in phase_run_oder:
77
+ report_urls: ReportUrls = report_data[phase]
78
+ job_report_url_info: ReportUrlInfo
79
+ for job_report_url_info in report_urls:
80
+ if not job_report_url_info:
81
+ continue
82
+ log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
83
+
84
+
68
85
  def report_on_run(
69
86
  phase_run_oder: OrderedPhases,
70
87
  job_run_metadatas: JobRunMetadatasByPhase,
71
88
  overall_runtime: timedelta,
72
- ):
73
- print("runem: reports:")
89
+ ) -> timedelta:
90
+ """Generate high-level reports AND prints out any reports returned by jobs.
91
+
92
+ IMPORTANT: returns the wall-clock time saved to the user.
93
+ """
94
+ log("reports:")
95
+
96
+ # First, collate all data, timing and reports
74
97
  timing_data: JobRunTimesByPhase = defaultdict(list)
75
98
  report_data: JobRunReportByPhase = defaultdict(list)
76
99
  phase: PhaseName
@@ -80,19 +103,21 @@ def report_on_run(
80
103
  for timing, reports in job_run_metadatas[phase]:
81
104
  timing_data[phase].append(timing)
82
105
  if reports:
106
+ # the job returned some report urls, record them against the
107
+ # job's phase
83
108
  report_data[phase].extend(reports["reportUrls"])
109
+
110
+ # Now plot the times on the terminal to give a visual report of the timing.
111
+ # Also, calculate the time saved by runem, a key selling-point metric
84
112
  time_saved: timedelta = _plot_times(
85
113
  overall_run_time=overall_runtime,
86
114
  phase_run_oder=phase_run_oder,
87
115
  timing_data=timing_data,
88
116
  )
89
- for phase in phase_run_oder:
90
- report_urls: ReportUrls = report_data[phase]
91
- job_report_url_info: ReportUrlInfo
92
- for job_report_url_info in report_urls:
93
- if not job_report_url_info:
94
- continue
95
- print(
96
- f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}"
97
- )
117
+
118
+ # Penultimate-ly print out the available reports grouped by run-phase.
119
+ _print_reports_by_phase(phase_run_oder, report_data)
120
+
121
+ # Return the key metric for runem, the wall-clock time saved to the user
122
+ # TODO: write this to disk
98
123
  return time_saved
runem/run_command.py CHANGED
@@ -5,6 +5,8 @@ from subprocess import STDOUT as SUBPROCESS_STDOUT
5
5
  from subprocess import CompletedProcess
6
6
  from subprocess import run as subprocess_run
7
7
 
8
+ from runem.log import log
9
+
8
10
  TERMINAL_WIDTH = 86
9
11
 
10
12
 
@@ -42,10 +44,10 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
42
44
  """Runs the given command, returning stdout or throwing on any error."""
43
45
  cmd_string: str = " ".join(cmd)
44
46
  if verbose:
45
- print(f"runem: running: start: {label}: {cmd_string}")
47
+ log(f"running: start: {label}: {cmd_string}")
46
48
  if valid_exit_ids is not None:
47
49
  valid_exit_strs = ",".join([str(exit_code) for exit_code in valid_exit_ids])
48
- print(f"runem:\tallowed return ids are: {valid_exit_strs}")
50
+ log(f"\tallowed return ids are: {valid_exit_strs}")
49
51
 
50
52
  if valid_exit_ids is None:
51
53
  valid_exit_ids = (0,)
@@ -60,13 +62,13 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
60
62
  run_env_as_string = " ".join(
61
63
  [f"{key}='{value}'" for key, value in run_env.items()]
62
64
  )
63
- print(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
65
+ log(f"RUN ENV OVERRIDES: {run_env_as_string } {cmd_string}")
64
66
 
65
67
  if env_overrides:
66
68
  env_overrides_as_string = " ".join(
67
69
  [f"{key}='{value}'" for key, value in env_overrides.items()]
68
70
  )
69
- print(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
71
+ log(f"ENV OVERRIDES: {env_overrides_as_string} {cmd_string}")
70
72
 
71
73
  env_overrides_dict = {}
72
74
  if env_overrides:
@@ -127,6 +129,6 @@ def run_command( # noqa: C901 # pylint: disable=too-many-branches
127
129
  assert process is not None
128
130
  cmd_stdout: str = get_stdout(process, prefix=label)
129
131
  if verbose:
130
- print(cmd_stdout)
131
- print(f"runem: running: done: {label}: {cmd_string}")
132
+ log(cmd_stdout)
133
+ log(f"running: done: {label}: {cmd_string}")
132
134
  return cmd_stdout
runem/runem.py CHANGED
@@ -23,19 +23,24 @@ import multiprocessing
23
23
  import os
24
24
  import pathlib
25
25
  import sys
26
+ import time
26
27
  import typing
27
28
  from collections import defaultdict
28
29
  from datetime import timedelta
29
30
  from itertools import repeat
31
+ from multiprocessing.managers import DictProxy, ValueProxy
30
32
  from timeit import default_timer as timer
31
33
 
34
+ from halo import Halo
35
+
32
36
  from runem.command_line import parse_args
33
37
  from runem.config import load_config
34
38
  from runem.config_metadata import ConfigMetadata
35
39
  from runem.config_parse import parse_config
36
40
  from runem.files import find_files
41
+ from runem.job_execute import job_execute
37
42
  from runem.job_filter import filter_jobs
38
- from runem.job_runner import job_runner
43
+ from runem.log import log
39
44
  from runem.report import report_on_run
40
45
  from runem.types import (
41
46
  Config,
@@ -69,11 +74,27 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
69
74
  config_metadata = parse_args(config_metadata, argv)
70
75
 
71
76
  if config_metadata.args.verbose:
72
- print(f"loaded config from {cfg_filepath}")
77
+ log(f"loaded config from {cfg_filepath}")
73
78
 
74
79
  return config_metadata
75
80
 
76
81
 
82
+ def _progress_updater(
83
+ label: str, running_jobs: typing.Dict[str, str], is_running: ValueProxy
84
+ ) -> None:
85
+ spinner = Halo(text="", spinner="dots")
86
+ spinner.start()
87
+
88
+ while is_running.value:
89
+ running_job_names: typing.List[str] = [
90
+ f"'{job}'" for job in sorted(list(running_jobs.values()))
91
+ ]
92
+ printable_jobs: str = ", ".join(running_job_names)
93
+ spinner.text = f"{label}: {printable_jobs}"
94
+ time.sleep(0.1)
95
+ spinner.stop()
96
+
97
+
77
98
  def _process_jobs(
78
99
  config_metadata: ConfigMetadata,
79
100
  file_lists: FilePathListLookup,
@@ -96,22 +117,38 @@ def _process_jobs(
96
117
  else multiprocessing.cpu_count()
97
118
  )
98
119
  num_concurrent_procs = min(num_concurrent_procs, len(jobs))
99
- print(
120
+ log(
100
121
  (
101
122
  f"Running '{phase}' with {num_concurrent_procs} workers "
102
123
  f"processing {len(jobs)} jobs"
103
124
  )
104
125
  )
105
- with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
106
- # use starmap so we can pass down the job-configs and the args and the files
107
- in_out_job_run_metadatas[phase] = pool.starmap(
108
- job_runner,
109
- zip(
110
- jobs,
111
- repeat(config_metadata),
112
- repeat(file_lists),
113
- ),
126
+
127
+ with multiprocessing.Manager() as manager:
128
+ running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
129
+ is_running: ValueProxy = manager.Value("b", True)
130
+
131
+ terminal_writer_process = multiprocessing.Process(
132
+ target=_progress_updater, args=(phase, running_jobs, is_running)
114
133
  )
134
+ terminal_writer_process.start()
135
+
136
+ try:
137
+ with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
138
+ # use starmap so we can pass down the job-configs and the args and the files
139
+ in_out_job_run_metadatas[phase] = pool.starmap(
140
+ job_execute,
141
+ zip(
142
+ jobs,
143
+ repeat(running_jobs),
144
+ repeat(config_metadata),
145
+ repeat(file_lists),
146
+ ),
147
+ )
148
+ finally:
149
+ # Signal the terminal_writer process to exit
150
+ is_running.value = False
151
+ terminal_writer_process.join()
115
152
 
116
153
 
117
154
  def _process_jobs_by_phase(
@@ -139,7 +176,7 @@ def _process_jobs_by_phase(
139
176
  continue
140
177
 
141
178
  if config_metadata.args.verbose:
142
- print(f"Running Phase {phase}")
179
+ log(f"Running Phase {phase}")
143
180
 
144
181
  _process_jobs(
145
182
  config_metadata, file_lists, in_out_job_run_metadatas, phase, jobs
@@ -158,11 +195,11 @@ def _main(
158
195
 
159
196
  file_lists: FilePathListLookup = find_files(config_metadata)
160
197
  assert file_lists
161
- print(f"found {len(file_lists)} batches, ", end="")
198
+ log(f"found {len(file_lists)} batches, ", end="")
162
199
  for tag in sorted(file_lists.keys()):
163
200
  file_list = file_lists[tag]
164
- print(f"{len(file_list)} '{tag}' files, ", end="")
165
- print() # new line
201
+ log(f"{len(file_list)} '{tag}' files, ", decorate=False, end="")
202
+ log(decorate=False) # new line
166
203
 
167
204
  filtered_jobs_by_phase: PhaseGroupedJobs = filter_jobs(
168
205
  config_metadata=config_metadata,
@@ -202,7 +239,7 @@ def timed_main(argv: typing.List[str]) -> None:
202
239
  end = timer()
203
240
  time_taken: timedelta = timedelta(seconds=end - start)
204
241
  time_saved = report_on_run(phase_run_oder, job_run_metadatas, time_taken)
205
- print(
242
+ log(
206
243
  (
207
244
  f"DONE: runem took: {time_taken.total_seconds()}s, "
208
245
  f"saving you {time_saved.total_seconds()}s"
runem/types.py CHANGED
@@ -18,8 +18,9 @@ JobPhases = typing.Set[str]
18
18
  JobTags = typing.Set[JobTag]
19
19
  PhaseName = str
20
20
  OrderedPhases = typing.Tuple[PhaseName, ...]
21
- ReportUrl = str | pathlib.Path
22
- ReportUrlInfo = typing.Tuple[str, ReportUrl]
21
+ ReportName = str
22
+ ReportUrl = typing.Union[str, pathlib.Path]
23
+ ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
23
24
  ReportUrls = typing.List[ReportUrlInfo]
24
25
 
25
26
 
@@ -1,11 +1,13 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: runem
3
- Version: 0.0.16
3
+ Version: 0.0.18
4
4
  Summary: Awesome runem created by lursight
5
5
  Home-page: https://github.com/lursight/runem/
6
6
  Author: lursight
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
+ Requires-Dist: halo
10
+ Requires-Dist: PyYAML
9
11
  Provides-Extra: test
10
12
  Requires-Dist: black ==23.11.0 ; extra == 'test'
11
13
  Requires-Dist: coverage ==7.3.2 ; extra == 'test'
@@ -23,7 +25,6 @@ Requires-Dist: pytest-cov ==4.1.0 ; extra == 'test'
23
25
  Requires-Dist: pytest-profiling ==1.7.0 ; extra == 'test'
24
26
  Requires-Dist: pytest-xdist ==3.3.1 ; extra == 'test'
25
27
  Requires-Dist: pytest ==7.4.3 ; extra == 'test'
26
- Requires-Dist: PyYAML ==6.0.1 ; extra == 'test'
27
28
  Requires-Dist: termplotlib ==0.3.9 ; extra == 'test'
28
29
  Requires-Dist: types-PyYAML ==6.0.12.12 ; extra == 'test'
29
30
  Requires-Dist: requests-mock ==1.10.0 ; extra == 'test'
@@ -0,0 +1,25 @@
1
+ runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
3
+ runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
4
+ runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
5
+ runem/command_line.py,sha256=lb1tvYOEVYpB_bUTRL8KxjLFuV4vKn8btFKm-28GMu0,9464
6
+ runem/config.py,sha256=h0r2WYUlAgdokUS1VU1sjFHgrN_cEq339ph4Kn1IBBM,1937
7
+ runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
8
+ runem/config_parse.py,sha256=gso5Lziw8yII5jGSx8fR-IAs7nxUzhzitnRAEXsg5f8,5157
9
+ runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
10
+ runem/job_execute.py,sha256=x-IGTG5KYnTJtCYm4jP8_WOQ_UM_BAiG5b3UUKp527A,3042
11
+ runem/job_filter.py,sha256=nltSRK1IKp2fuTCHOZFcii0FBL-R3RKvrnyWXTPjoMw,3478
12
+ runem/job_wrapper_python.py,sha256=TG9bzDe1dgTnsrXrCPd6LzN8qbMkH4rEv_vXC_zlJc0,4226
13
+ runem/log.py,sha256=m1lI4V8ygM53pY4Go4eEzvEJY8srVoItxUNhdp_Vrqg,314
14
+ runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
+ runem/report.py,sha256=AyD2yT_8ODo5QRuFkVFIecp6ycT4XElJbleWviZbCfI,4032
16
+ runem/run_command.py,sha256=6QxH8pb4Lug-d0MCP2lBKq2TJlQloxRlq0hYJM6IScc,4528
17
+ runem/runem.py,sha256=Q5ryZtArAGuTlHF07aobJkRIzurKIUsMSUPzEksL6LY,8227
18
+ runem/types.py,sha256=Hw7G7ut7yNaoc3XiDNKeM-cAU-DDu3fDOne13FrutS0,5374
19
+ runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
20
+ runem-0.0.18.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
21
+ runem-0.0.18.dist-info/METADATA,sha256=VzkG4gPaopCtwCnEx1XRGLk9iNWpLPjl4x8BY-JvSSA,15909
22
+ runem-0.0.18.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
23
+ runem-0.0.18.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
24
+ runem-0.0.18.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
25
+ runem-0.0.18.dist-info/RECORD,,
@@ -1,24 +0,0 @@
1
- runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
- runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
3
- runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
4
- runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
5
- runem/command_line.py,sha256=MD3tAdXO7WNKRabwSJwBks1az2mYyTAxifxX-vGA8yc,9425
6
- runem/config.py,sha256=jtpMOS-7J0YFkQmXpDy3CPtohEd4HBHGmjPTzkL0Tyw,1913
7
- runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
8
- runem/config_parse.py,sha256=eA5oAW8T7-kv6TnNEsnhIvRP0AoPItufwaTTaMr7yZ0,5140
9
- runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
10
- runem/job_filter.py,sha256=cNlc9UVi6S1O9Ui95GH3n4dFEYRiGJ4KUIt5xPDQWWw,3313
11
- runem/job_function_python.py,sha256=iNRKQp0Rlkh1VektOxUSa_KqrrY0yB7jDChcRt-s6oQ,3754
12
- runem/job_runner.py,sha256=cCpZg_1LOMalQoY23_5A7R8vu7aaNjliZL4q8qxdIz0,2834
13
- runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
14
- runem/report.py,sha256=S-KV70gMBUlxjqtzQkCqSIDhsRk6X20i9RKLv6-Y5vk,3104
15
- runem/run_command.py,sha256=UPwQU7eC5FADcX3aPrNP8kp-g5YONBdwwjsz-NCaMno,4533
16
- runem/runem.py,sha256=LEccvElpxpuKM6Bl4gvrc1-ZlMkSB8b7K13l5nyzNP4,6941
17
- runem/types.py,sha256=IxUDGHOQZUc86OPjDHDKwrcBFr78v-MNvTpPwUzWoaI,5337
18
- runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
19
- runem-0.0.16.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
20
- runem-0.0.16.dist-info/METADATA,sha256=P-__qoOFtuC8JZU7uh3hEBeN_U7kHW7U8hGY9q_5A6U,15915
21
- runem-0.0.16.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
22
- runem-0.0.16.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
23
- runem-0.0.16.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
24
- runem-0.0.16.dist-info/RECORD,,
File without changes