runem 0.0.29__py3-none-any.whl → 0.0.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runem/VERSION +1 -1
- runem/cli.py +1 -0
- runem/command_line.py +33 -8
- runem/config.py +58 -9
- runem/config_metadata.py +8 -0
- runem/config_parse.py +188 -13
- runem/files.py +32 -7
- runem/hook_manager.py +116 -0
- runem/job_execute.py +22 -21
- runem/job_filter.py +2 -2
- runem/job_runner_simple_command.py +7 -1
- runem/job_wrapper.py +11 -5
- runem/job_wrapper_python.py +7 -7
- runem/log.py +8 -0
- runem/report.py +18 -14
- runem/runem.py +30 -13
- runem/types.py +47 -4
- {runem-0.0.29.dist-info → runem-0.0.30.dist-info}/METADATA +18 -28
- runem-0.0.30.dist-info/RECORD +33 -0
- runem-0.0.29.dist-info/RECORD +0 -32
- {runem-0.0.29.dist-info → runem-0.0.30.dist-info}/LICENSE +0 -0
- {runem-0.0.29.dist-info → runem-0.0.30.dist-info}/WHEEL +0 -0
- {runem-0.0.29.dist-info → runem-0.0.30.dist-info}/entry_points.txt +0 -0
- {runem-0.0.29.dist-info → runem-0.0.30.dist-info}/top_level.txt +0 -0
runem/job_execute.py
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
import inspect
|
2
1
|
import os
|
3
2
|
import pathlib
|
4
3
|
import typing
|
@@ -10,7 +9,7 @@ from runem.config_metadata import ConfigMetadata
|
|
10
9
|
from runem.informative_dict import ReadOnlyInformativeDict
|
11
10
|
from runem.job import Job
|
12
11
|
from runem.job_wrapper import get_job_wrapper
|
13
|
-
from runem.log import log
|
12
|
+
from runem.log import error, log
|
14
13
|
from runem.types import (
|
15
14
|
FilePathListLookup,
|
16
15
|
JobConfig,
|
@@ -27,6 +26,7 @@ def job_execute_inner(
|
|
27
26
|
job_config: JobConfig,
|
28
27
|
config_metadata: ConfigMetadata,
|
29
28
|
file_lists: FilePathListLookup,
|
29
|
+
**kwargs: typing.Any,
|
30
30
|
) -> typing.Tuple[JobTiming, JobReturn]:
|
31
31
|
"""Wrapper for running a job inside a sub-process.
|
32
32
|
|
@@ -73,31 +73,26 @@ def job_execute_inner(
|
|
73
73
|
os.chdir(root_path)
|
74
74
|
|
75
75
|
start = timer()
|
76
|
-
func_signature = inspect.signature(function)
|
77
76
|
if config_metadata.args.verbose:
|
78
77
|
log(f"job: running: '{Job.get_job_name(job_config)}'")
|
79
78
|
reports: JobReturn
|
80
79
|
try:
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
label=Job.get_job_name(job_config),
|
94
|
-
job=job_config,
|
95
|
-
record_sub_job_time=_record_sub_job_time,
|
96
|
-
)
|
80
|
+
reports = function(
|
81
|
+
options=ReadOnlyInformativeDict(config_metadata.options), # type: ignore
|
82
|
+
file_list=file_list,
|
83
|
+
procs=config_metadata.args.procs,
|
84
|
+
root_path=root_path,
|
85
|
+
verbose=config_metadata.args.verbose,
|
86
|
+
# unpack useful data points from the job_config
|
87
|
+
label=Job.get_job_name(job_config),
|
88
|
+
job=job_config,
|
89
|
+
record_sub_job_time=_record_sub_job_time,
|
90
|
+
**kwargs,
|
91
|
+
)
|
97
92
|
except BaseException: # pylint: disable=broad-exception-caught
|
98
93
|
# log that we hit an error on this job and re-raise
|
99
94
|
log(decorate=False)
|
100
|
-
|
95
|
+
error(f"job: job '{Job.get_job_name(job_config)}' failed to complete!")
|
101
96
|
# re-raise
|
102
97
|
raise
|
103
98
|
|
@@ -114,6 +109,7 @@ def job_execute(
|
|
114
109
|
running_jobs: typing.Dict[str, str],
|
115
110
|
config_metadata: ConfigMetadata,
|
116
111
|
file_lists: FilePathListLookup,
|
112
|
+
**kwargs: typing.Any,
|
117
113
|
) -> typing.Tuple[JobTiming, JobReturn]:
|
118
114
|
"""Thin-wrapper around job_execute_inner needed for mocking in tests.
|
119
115
|
|
@@ -121,6 +117,11 @@ def job_execute(
|
|
121
117
|
"""
|
122
118
|
this_id: str = str(uuid.uuid4())
|
123
119
|
running_jobs[this_id] = Job.get_job_name(job_config)
|
124
|
-
results = job_execute_inner(
|
120
|
+
results = job_execute_inner(
|
121
|
+
job_config,
|
122
|
+
config_metadata,
|
123
|
+
file_lists,
|
124
|
+
**kwargs,
|
125
|
+
)
|
125
126
|
del running_jobs[this_id]
|
126
127
|
return results
|
runem/job_filter.py
CHANGED
@@ -35,7 +35,7 @@ def _should_filter_out_by_tags(
|
|
35
35
|
if verbose:
|
36
36
|
log(
|
37
37
|
(
|
38
|
-
f"not running job '{job
|
38
|
+
f"not running job '{Job.get_job_name(job)}' because it doesn't have "
|
39
39
|
f"any of the following tags: {printable_set(tags)}"
|
40
40
|
)
|
41
41
|
)
|
@@ -46,7 +46,7 @@ def _should_filter_out_by_tags(
|
|
46
46
|
if verbose:
|
47
47
|
log(
|
48
48
|
(
|
49
|
-
f"not running job '{job
|
49
|
+
f"not running job '{Job.get_job_name(job)}' because it contains the "
|
50
50
|
f"following tags: {printable_set(has_tags_to_avoid)}"
|
51
51
|
)
|
52
52
|
)
|
@@ -5,6 +5,12 @@ from runem.run_command import run_command
|
|
5
5
|
from runem.types import JobConfig
|
6
6
|
|
7
7
|
|
8
|
+
def validate_simple_command(command_string: str) -> typing.List[str]:
|
9
|
+
# use shlex to handle parsing of the command string, a non-trivial problem.
|
10
|
+
split_command: typing.List[str] = shlex.split(command_string)
|
11
|
+
return split_command
|
12
|
+
|
13
|
+
|
8
14
|
def job_runner_simple_command(
|
9
15
|
**kwargs: typing.Any,
|
10
16
|
) -> None:
|
@@ -17,7 +23,7 @@ def job_runner_simple_command(
|
|
17
23
|
command_string: str = job_config["command"]
|
18
24
|
|
19
25
|
# use shlex to handle parsing of the command string, a non-trivial problem.
|
20
|
-
result =
|
26
|
+
result = validate_simple_command(command_string)
|
21
27
|
|
22
28
|
# preserve quotes for consistent handling of strings and avoid the "word
|
23
29
|
# splitting" problem for unix-like shells.
|
runem/job_wrapper.py
CHANGED
@@ -1,19 +1,25 @@
|
|
1
1
|
import pathlib
|
2
2
|
|
3
|
-
from runem.job_runner_simple_command import
|
3
|
+
from runem.job_runner_simple_command import (
|
4
|
+
job_runner_simple_command,
|
5
|
+
validate_simple_command,
|
6
|
+
)
|
4
7
|
from runem.job_wrapper_python import get_job_wrapper_py_func
|
5
|
-
from runem.types import
|
8
|
+
from runem.types import JobFunction, JobWrapper
|
6
9
|
|
7
10
|
|
8
|
-
def get_job_wrapper(
|
11
|
+
def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobFunction:
|
9
12
|
"""Given a job-description determines the job-runner, returning it as a function.
|
10
13
|
|
11
14
|
NOTE: Side-effects: also re-addressed the job-config in the case of functions see
|
12
15
|
get_job_function.
|
13
16
|
"""
|
14
|
-
if "command" in
|
17
|
+
if "command" in job_wrapper:
|
18
|
+
# validate that the command is "understandable" and usable.
|
19
|
+
command_string: str = job_wrapper["command"]
|
20
|
+
validate_simple_command(command_string)
|
15
21
|
return job_runner_simple_command # type: ignore # NO_COMMIT
|
16
22
|
|
17
23
|
# if we do not have a simple command address assume we have just an addressed
|
18
24
|
# function
|
19
|
-
return get_job_wrapper_py_func(
|
25
|
+
return get_job_wrapper_py_func(job_wrapper, cfg_filepath)
|
runem/job_wrapper_python.py
CHANGED
@@ -3,7 +3,7 @@ import sys
|
|
3
3
|
from importlib.util import module_from_spec
|
4
4
|
from importlib.util import spec_from_file_location as module_spec_from_file_location
|
5
5
|
|
6
|
-
from runem.types import FunctionNotFound,
|
6
|
+
from runem.types import FunctionNotFound, JobFunction, JobWrapper
|
7
7
|
|
8
8
|
|
9
9
|
def _load_python_function_from_module(
|
@@ -86,22 +86,22 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
|
|
86
86
|
|
87
87
|
|
88
88
|
def get_job_wrapper_py_func(
|
89
|
-
|
89
|
+
job_wrapper: JobWrapper, cfg_filepath: pathlib.Path
|
90
90
|
) -> JobFunction:
|
91
91
|
"""For a job, dynamically loads the associated python job-function.
|
92
92
|
|
93
93
|
Side-effects: also re-addressed the job-config.
|
94
94
|
"""
|
95
|
-
function_to_load: str =
|
95
|
+
function_to_load: str = job_wrapper["addr"]["function"]
|
96
96
|
try:
|
97
97
|
module_file_path: pathlib.Path = _find_job_module(
|
98
|
-
cfg_filepath,
|
98
|
+
cfg_filepath, job_wrapper["addr"]["file"]
|
99
99
|
)
|
100
100
|
except FunctionNotFound as err:
|
101
101
|
raise FunctionNotFound(
|
102
102
|
(
|
103
|
-
|
104
|
-
f"job.addr.file '{
|
103
|
+
"runem failed to find "
|
104
|
+
f"job.addr.file '{job_wrapper['addr']['file']}' looking for "
|
105
105
|
f"job.addr.function '{function_to_load}'"
|
106
106
|
)
|
107
107
|
) from err
|
@@ -118,5 +118,5 @@ def get_job_wrapper_py_func(
|
|
118
118
|
)
|
119
119
|
|
120
120
|
# re-write the job-config file-path for the module with the one that worked
|
121
|
-
|
121
|
+
job_wrapper["addr"]["file"] = str(module_file_path)
|
122
122
|
return function
|
runem/log.py
CHANGED
@@ -14,3 +14,11 @@ def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None)
|
|
14
14
|
# print in a blocking manner, waiting for system resources to free up if a
|
15
15
|
# runem job is contending on stdout or similar.
|
16
16
|
blocking_print(msg, end=end)
|
17
|
+
|
18
|
+
|
19
|
+
def warn(msg: str) -> None:
|
20
|
+
log(f"WARNING: {msg}")
|
21
|
+
|
22
|
+
|
23
|
+
def error(msg: str) -> None:
|
24
|
+
log(f"ERROR: {msg}")
|
runem/report.py
CHANGED
@@ -43,7 +43,7 @@ def _align_bar_graphs_workaround(original_text: str) -> str:
|
|
43
43
|
return formatted_text
|
44
44
|
|
45
45
|
|
46
|
-
def
|
46
|
+
def replace_bar_graph_characters(text: str, end_str: str, replace_char: str) -> str:
|
47
47
|
"""Replaces block characters in lines containing `end_str` with give char.
|
48
48
|
|
49
49
|
Args:
|
@@ -56,16 +56,19 @@ def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
|
|
56
56
|
"""
|
57
57
|
# Define the block character and its light shade replacement
|
58
58
|
block_chars = (
|
59
|
-
"
|
59
|
+
"▏▎▍▋▊▉█▌▐▄▀─" # Extend this string with any additional block characters you use
|
60
|
+
"░·" # also include the chars we might replace with for special bars
|
60
61
|
)
|
61
62
|
|
62
63
|
text_lines: typing.List[str] = text.split("\n")
|
63
64
|
|
64
65
|
# Process each line, replacing block characters if `end_str` is present
|
65
66
|
modified_lines = [
|
66
|
-
|
67
|
-
|
68
|
-
|
67
|
+
(
|
68
|
+
line.translate(str.maketrans(block_chars, replace_char * len(block_chars)))
|
69
|
+
if end_str in line
|
70
|
+
else line
|
71
|
+
)
|
69
72
|
for line in text_lines
|
70
73
|
]
|
71
74
|
|
@@ -74,12 +77,12 @@ def _replace_bar_characters(text: str, end_str: str, replace_char: str) -> str:
|
|
74
77
|
|
75
78
|
def _semi_shade_phase_totals(text: str) -> str:
|
76
79
|
light_shade_char = "░"
|
77
|
-
return
|
80
|
+
return replace_bar_graph_characters(text, "(user-time)", light_shade_char)
|
78
81
|
|
79
82
|
|
80
83
|
def _dot_jobs(text: str) -> str:
|
81
84
|
dot_char = "·"
|
82
|
-
return
|
85
|
+
return replace_bar_graph_characters(text, "(+)", dot_char)
|
83
86
|
|
84
87
|
|
85
88
|
def _plot_times(
|
@@ -103,8 +106,8 @@ def _plot_times(
|
|
103
106
|
|
104
107
|
for idx, phase in enumerate(phase_run_oder):
|
105
108
|
not_last_phase: bool = idx < len(phase_run_oder) - 1
|
106
|
-
utf8_phase = "├" if not_last_phase else "└"
|
107
|
-
utf8_phase_group = "│" if not_last_phase else "
|
109
|
+
utf8_phase = " ├" if not_last_phase else " └"
|
110
|
+
utf8_phase_group = " │" if not_last_phase else " "
|
108
111
|
# log(f"Phase '{phase}' jobs took:")
|
109
112
|
phase_start_idx = len(labels)
|
110
113
|
|
@@ -121,9 +124,11 @@ def _plot_times(
|
|
121
124
|
|
122
125
|
runem_app_timing: typing.List[JobTiming] = timing_data["_app"]
|
123
126
|
job_metadata: JobTiming
|
124
|
-
for job_metadata in reversed(runem_app_timing):
|
127
|
+
for idx, job_metadata in enumerate(reversed(runem_app_timing)):
|
128
|
+
last_group: bool = idx == 0 # reverse sorted
|
129
|
+
utf8_group = "├" if not last_group else "└"
|
125
130
|
job_label, job_time_total = job_metadata["job"]
|
126
|
-
labels.insert(0, f"
|
131
|
+
labels.insert(0, f"{utf8_group}runem.{job_label}")
|
127
132
|
times.insert(0, job_time_total.total_seconds())
|
128
133
|
labels.insert(0, "runem (total wall-clock)")
|
129
134
|
times.insert(0, wall_clock_for_runem_main.total_seconds())
|
@@ -175,7 +180,7 @@ def _gen_jobs_report(
|
|
175
180
|
utf8_job = "├" if not_last else "└"
|
176
181
|
utf8_sub_jobs = "│" if not_last else " "
|
177
182
|
job_label, job_time_total = job_timing["job"]
|
178
|
-
job_bar_label: str = f"{
|
183
|
+
job_bar_label: str = f"{job_label}"
|
179
184
|
labels.append(f"{utf8_phase_group}{utf8_job}{job_bar_label}")
|
180
185
|
times.append(job_time_total.total_seconds())
|
181
186
|
job_time_sum += job_time_total
|
@@ -191,8 +196,7 @@ def _gen_jobs_report(
|
|
191
196
|
if idx == len(sub_command_times) - 1:
|
192
197
|
sub_utf8 = "└"
|
193
198
|
labels.append(
|
194
|
-
f"{utf8_phase_group}{utf8_sub_jobs}{sub_utf8}{
|
195
|
-
f".{sub_job_label} (+)"
|
199
|
+
f"{utf8_phase_group}{utf8_sub_jobs}{sub_utf8}{sub_job_label} (+)"
|
196
200
|
)
|
197
201
|
times.append(sub_job_time.total_seconds())
|
198
202
|
return job_time_sum
|
runem/runem.py
CHANGED
@@ -34,18 +34,19 @@ from timeit import default_timer as timer
|
|
34
34
|
from halo import Halo
|
35
35
|
|
36
36
|
from runem.command_line import parse_args
|
37
|
-
from runem.config import
|
37
|
+
from runem.config import load_project_config, load_user_configs
|
38
38
|
from runem.config_metadata import ConfigMetadata
|
39
|
-
from runem.config_parse import
|
39
|
+
from runem.config_parse import load_config_metadata
|
40
40
|
from runem.files import find_files
|
41
41
|
from runem.job import Job
|
42
42
|
from runem.job_execute import job_execute
|
43
43
|
from runem.job_filter import filter_jobs
|
44
|
-
from runem.log import log
|
44
|
+
from runem.log import error, log, warn
|
45
45
|
from runem.report import report_on_run
|
46
46
|
from runem.types import (
|
47
47
|
Config,
|
48
48
|
FilePathListLookup,
|
49
|
+
HookName,
|
49
50
|
JobReturn,
|
50
51
|
JobRunMetadata,
|
51
52
|
JobRunMetadatasByPhase,
|
@@ -68,8 +69,11 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
|
|
68
69
|
"""
|
69
70
|
config: Config
|
70
71
|
cfg_filepath: pathlib.Path
|
71
|
-
config, cfg_filepath =
|
72
|
-
|
72
|
+
config, cfg_filepath = load_project_config()
|
73
|
+
user_configs: typing.List[typing.Tuple[Config, pathlib.Path]] = load_user_configs()
|
74
|
+
config_metadata: ConfigMetadata = load_config_metadata(
|
75
|
+
config, cfg_filepath, user_configs, verbose=("--verbose" in argv)
|
76
|
+
)
|
73
77
|
|
74
78
|
# Now we parse the cli arguments extending them with information from the
|
75
79
|
# .runem.yml config.
|
@@ -263,18 +267,21 @@ def _process_jobs_by_phase(
|
|
263
267
|
)
|
264
268
|
if failure_exception is not None:
|
265
269
|
if config_metadata.args.verbose:
|
266
|
-
|
270
|
+
error(f"running phase {phase}: aborting run")
|
267
271
|
return failure_exception
|
268
272
|
|
269
273
|
# ALl phases completed aok.
|
270
274
|
return None
|
271
275
|
|
272
276
|
|
277
|
+
MainReturnType = typing.Tuple[
|
278
|
+
ConfigMetadata, JobRunMetadatasByPhase, typing.Optional[BaseException]
|
279
|
+
]
|
280
|
+
|
281
|
+
|
273
282
|
def _main(
|
274
283
|
argv: typing.List[str],
|
275
|
-
) ->
|
276
|
-
OrderedPhases, JobRunMetadatasByPhase, typing.Optional[BaseException]
|
277
|
-
]:
|
284
|
+
) -> MainReturnType:
|
278
285
|
start = timer()
|
279
286
|
|
280
287
|
config_metadata: ConfigMetadata = _determine_run_parameters(argv)
|
@@ -283,7 +290,10 @@ def _main(
|
|
283
290
|
os.chdir(config_metadata.cfg_filepath.parent)
|
284
291
|
|
285
292
|
file_lists: FilePathListLookup = find_files(config_metadata)
|
286
|
-
|
293
|
+
if not file_lists:
|
294
|
+
warn("no files found")
|
295
|
+
return (config_metadata, {}, None)
|
296
|
+
|
287
297
|
if config_metadata.args.verbose:
|
288
298
|
log(f"found {len(file_lists)} batches, ", end="")
|
289
299
|
for tag in sorted(file_lists.keys()):
|
@@ -322,7 +332,7 @@ def _main(
|
|
322
332
|
phase_run_report: JobReturn = None
|
323
333
|
phase_run_metadata: JobRunMetadata = (phase_run_timing, phase_run_report)
|
324
334
|
job_run_metadatas["_app"].append(phase_run_metadata)
|
325
|
-
return config_metadata
|
335
|
+
return config_metadata, job_run_metadatas, failure_exception
|
326
336
|
|
327
337
|
|
328
338
|
def timed_main(argv: typing.List[str]) -> None:
|
@@ -332,10 +342,11 @@ def timed_main(argv: typing.List[str]) -> None:
|
|
332
342
|
are representative.
|
333
343
|
"""
|
334
344
|
start = timer()
|
335
|
-
|
345
|
+
config_metadata: ConfigMetadata
|
336
346
|
job_run_metadatas: JobRunMetadatasByPhase
|
337
347
|
failure_exception: typing.Optional[BaseException]
|
338
|
-
|
348
|
+
config_metadata, job_run_metadatas, failure_exception = _main(argv)
|
349
|
+
phase_run_oder: OrderedPhases = config_metadata.phases
|
339
350
|
end = timer()
|
340
351
|
time_taken: timedelta = timedelta(seconds=end - start)
|
341
352
|
wall_clock_time_saved: timedelta
|
@@ -354,6 +365,12 @@ def timed_main(argv: typing.List[str]) -> None:
|
|
354
365
|
)
|
355
366
|
)
|
356
367
|
|
368
|
+
config_metadata.hook_manager.invoke_hooks(
|
369
|
+
hook_name=HookName.ON_EXIT,
|
370
|
+
config_metadata=config_metadata,
|
371
|
+
wall_clock_time_saved=wall_clock_time_saved,
|
372
|
+
)
|
373
|
+
|
357
374
|
if failure_exception is not None:
|
358
375
|
# we got a failure somewhere, now that we've reported the timings we
|
359
376
|
# re-raise.
|
runem/types.py
CHANGED
@@ -2,6 +2,7 @@ import argparse
|
|
2
2
|
import pathlib
|
3
3
|
import typing
|
4
4
|
from datetime import timedelta
|
5
|
+
from enum import Enum
|
5
6
|
|
6
7
|
from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
|
7
8
|
|
@@ -26,6 +27,15 @@ ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
|
|
26
27
|
ReportUrls = typing.List[ReportUrlInfo]
|
27
28
|
|
28
29
|
|
30
|
+
class HookName(Enum):
|
31
|
+
# at exit
|
32
|
+
ON_EXIT = "on-exit"
|
33
|
+
# before all tasks are run, after config is read
|
34
|
+
# BEFORE_ALL = "before-all"
|
35
|
+
# after all tasks are done, before reporting
|
36
|
+
# AFTER_ALL = "after-all"
|
37
|
+
|
38
|
+
|
29
39
|
class JobReturnData(typing.TypedDict, total=False):
|
30
40
|
"""A dict that defines job result to be reported to the user."""
|
31
41
|
|
@@ -125,7 +135,14 @@ class JobWhen(typing.TypedDict, total=False):
|
|
125
135
|
phase: PhaseName # the phase when the job should be run
|
126
136
|
|
127
137
|
|
128
|
-
class
|
138
|
+
class JobWrapper(typing.TypedDict, total=False):
|
139
|
+
"""A base-type for jobs, hooks, and things that can be invoked."""
|
140
|
+
|
141
|
+
addr: JobAddressConfig # which callable to call
|
142
|
+
command: str # a one-liner command to be run
|
143
|
+
|
144
|
+
|
145
|
+
class JobConfig(JobWrapper, total=False):
|
129
146
|
"""A dict that defines a job to be run.
|
130
147
|
|
131
148
|
It consists of the label, address, context and filter information
|
@@ -134,8 +151,6 @@ class JobConfig(typing.TypedDict, total=False):
|
|
134
151
|
"""
|
135
152
|
|
136
153
|
label: JobName # the name of the job
|
137
|
-
addr: JobAddressConfig # which callable to call
|
138
|
-
command: str # a one-liner command to be run
|
139
154
|
ctx: typing.Optional[JobContextConfig] # how to call the callable
|
140
155
|
when: JobWhen # when to call the job
|
141
156
|
|
@@ -193,6 +208,30 @@ class GlobalSerialisedConfig(typing.TypedDict):
|
|
193
208
|
config: GlobalConfig
|
194
209
|
|
195
210
|
|
211
|
+
class HookConfig(JobWrapper, total=False):
|
212
|
+
"""Specification for hooks.
|
213
|
+
|
214
|
+
Like JobConfig with use addr or command to specify what to execute.
|
215
|
+
"""
|
216
|
+
|
217
|
+
hook_name: HookName # the hook for when this is called
|
218
|
+
|
219
|
+
|
220
|
+
Hooks = typing.DefaultDict[HookName, typing.List[HookConfig]]
|
221
|
+
|
222
|
+
# A dictionary to hold hooks, with hook names as keys
|
223
|
+
HooksStore = typing.Dict[HookName, typing.List[HookConfig]]
|
224
|
+
|
225
|
+
|
226
|
+
class HookSerialisedConfig(typing.TypedDict):
|
227
|
+
"""Intended to make reading a config file easier.
|
228
|
+
|
229
|
+
Also, unlike JobSerialisedConfig, this type may not actually help readability.
|
230
|
+
"""
|
231
|
+
|
232
|
+
hook: HookConfig
|
233
|
+
|
234
|
+
|
196
235
|
class JobSerialisedConfig(typing.TypedDict):
|
197
236
|
"""Makes serialised configs easier to read.
|
198
237
|
|
@@ -204,6 +243,10 @@ class JobSerialisedConfig(typing.TypedDict):
|
|
204
243
|
job: JobConfig
|
205
244
|
|
206
245
|
|
207
|
-
ConfigNodes = typing.Union[
|
246
|
+
ConfigNodes = typing.Union[
|
247
|
+
GlobalSerialisedConfig, JobSerialisedConfig, HookSerialisedConfig
|
248
|
+
]
|
208
249
|
# The config format as it is serialised to/from disk
|
209
250
|
Config = typing.List[ConfigNodes]
|
251
|
+
|
252
|
+
UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: runem
|
3
|
-
Version: 0.0.
|
3
|
+
Version: 0.0.30
|
4
4
|
Summary: Awesome runem created by lursight
|
5
5
|
Home-page: https://github.com/lursight/runem/
|
6
6
|
Author: lursight
|
@@ -10,26 +10,26 @@ Requires-Dist: halo
|
|
10
10
|
Requires-Dist: packaging
|
11
11
|
Requires-Dist: PyYAML
|
12
12
|
Provides-Extra: test
|
13
|
-
Requires-Dist: black ==
|
14
|
-
Requires-Dist: coverage ==7.
|
13
|
+
Requires-Dist: black ==24.3.0 ; extra == 'test'
|
14
|
+
Requires-Dist: coverage ==7.4.4 ; extra == 'test'
|
15
15
|
Requires-Dist: docformatter ==1.7.5 ; extra == 'test'
|
16
16
|
Requires-Dist: flake8-bugbear ==24.2.6 ; extra == 'test'
|
17
17
|
Requires-Dist: flake8 ==7.0.0 ; extra == 'test'
|
18
18
|
Requires-Dist: gitchangelog ==3.0.4 ; extra == 'test'
|
19
|
-
Requires-Dist: isort ==5.
|
19
|
+
Requires-Dist: isort ==5.13.2 ; extra == 'test'
|
20
20
|
Requires-Dist: mkdocs ==1.5.3 ; extra == 'test'
|
21
|
-
Requires-Dist: mypy ==1.
|
21
|
+
Requires-Dist: mypy ==1.9.0 ; extra == 'test'
|
22
22
|
Requires-Dist: pydocstyle ==6.3.0 ; extra == 'test'
|
23
|
-
Requires-Dist: pylint ==3.0
|
23
|
+
Requires-Dist: pylint ==3.1.0 ; extra == 'test'
|
24
24
|
Requires-Dist: pylama ==8.4.1 ; extra == 'test'
|
25
25
|
Requires-Dist: pytest-cov ==4.1.0 ; extra == 'test'
|
26
26
|
Requires-Dist: pytest-profiling ==1.7.0 ; extra == 'test'
|
27
|
-
Requires-Dist: pytest-xdist ==3.
|
28
|
-
Requires-Dist: pytest ==
|
27
|
+
Requires-Dist: pytest-xdist ==3.5.0 ; extra == 'test'
|
28
|
+
Requires-Dist: pytest ==8.1.1 ; extra == 'test'
|
29
29
|
Requires-Dist: setuptools ; extra == 'test'
|
30
30
|
Requires-Dist: termplotlib ==0.3.9 ; extra == 'test'
|
31
|
-
Requires-Dist: types-PyYAML ==6.0.12.
|
32
|
-
Requires-Dist: requests-mock ==1.
|
31
|
+
Requires-Dist: types-PyYAML ==6.0.12.20240311 ; extra == 'test'
|
32
|
+
Requires-Dist: requests-mock ==1.11.0 ; extra == 'test'
|
33
33
|
Requires-Dist: types-setuptools ; extra == 'test'
|
34
34
|
|
35
35
|
# Run 'em
|
@@ -40,7 +40,7 @@ Requires-Dist: types-setuptools ; extra == 'test'
|
|
40
40
|
|
41
41
|
## 1. Overview
|
42
42
|
|
43
|
-
The core objective of Run'em's is to minimize the wall-clock time required for
|
43
|
+
The core objective of Run'em's is to minimize the wall-clock time required for running checks, supporting [shift-left](https://en.wikipedia.org/wiki/Shift-left_testing). Overall it is designed to enhance iteration speed and boost developer productivity.
|
44
44
|
|
45
45
|
`runem` is also designed to be easy to learn and simple to use, but `runem` also has many powerful tools for advanced users.
|
46
46
|
|
@@ -173,7 +173,7 @@ Here's a simple setup for a python project.
|
|
173
173
|
type: bool
|
174
174
|
desc: formats docs and comments in whatever job can do so
|
175
175
|
- option:
|
176
|
-
name:
|
176
|
+
name: check-only
|
177
177
|
alias: check
|
178
178
|
default: false
|
179
179
|
type: bool
|
@@ -247,11 +247,11 @@ def _job_py_code_reformat(
|
|
247
247
|
docformatter_extra_args = [
|
248
248
|
"--in-place",
|
249
249
|
]
|
250
|
-
if
|
250
|
+
if options["check-only"]:
|
251
251
|
extra_args.append("--check")
|
252
252
|
docformatter_extra_args = [] # --inplace is not compatible with --check
|
253
253
|
|
254
|
-
if
|
254
|
+
if options["black"]:
|
255
255
|
black_cmd = [
|
256
256
|
"python3",
|
257
257
|
"-m",
|
@@ -262,7 +262,7 @@ def _job_py_code_reformat(
|
|
262
262
|
kwargs["label"] = f"{label} black"
|
263
263
|
run_command(cmd=black_cmd, **kwargs)
|
264
264
|
|
265
|
-
if
|
265
|
+
if options["docformatter"]:
|
266
266
|
docformatter_cmd = [
|
267
267
|
"python3",
|
268
268
|
"-m",
|
@@ -279,7 +279,7 @@ def _job_py_code_reformat(
|
|
279
279
|
0, # no work/change required
|
280
280
|
3, # no errors, but code was reformatted
|
281
281
|
)
|
282
|
-
if
|
282
|
+
if options["check-only"]:
|
283
283
|
# in check it is ONLY ok if no work/change was required
|
284
284
|
allowed_exits = (0,)
|
285
285
|
kwargs["label"] = f"{label} docformatter"
|
@@ -469,8 +469,8 @@ usage: runem.py [-h] [--jobs JOBS [JOBS ...]] [--not-jobs JOBS_EXCLUDED [JOBS_EX
|
|
469
469
|
[--not-phases PHASES_EXCLUDED [PHASES_EXCLUDED ...]] [--tags TAGS [TAGS ...]] [--not-tags TAGS_EXCLUDED [TAGS_EXCLUDED ...]]
|
470
470
|
[--black] [--no-black] [--check-only] [--no-check-only] [--coverage] [--no-coverage] [--docformatter] [--no-docformatter]
|
471
471
|
[--generate-call-graphs] [--no-generate-call-graphs] [--install-deps] [--no-install-deps] [--isort] [--no-isort] [--profile]
|
472
|
-
[--no-profile] [--
|
473
|
-
[--
|
472
|
+
[--no-profile] [--unit-test] [--no-unit-test]
|
473
|
+
[--call-graphs | --no-call-graphs]
|
474
474
|
[--procs PROCS] [--root ROOT_DIR] [--verbose | --no-verbose | -v]
|
475
475
|
|
476
476
|
Runs the Lursight Lang test-suite
|
@@ -524,18 +524,8 @@ job-param overrides:
|
|
524
524
|
--no-isort turn off allow/disallows isort from running on python files
|
525
525
|
--profile generate profile information in jobs that can
|
526
526
|
--no-profile turn off generate profile information in jobs that can
|
527
|
-
--update-snapshots update snapshots in jobs that can update data snapshots
|
528
|
-
--no-update-snapshots
|
529
|
-
turn off update snapshots in jobs that can update data snapshots
|
530
527
|
--unit-test run unit tests
|
531
528
|
--no-unit-test turn off run unit tests
|
532
|
-
--unit-test-firebase-data
|
533
|
-
run unit tests for the firebase function's data
|
534
|
-
--no-unit-test-firebase-data
|
535
|
-
turn off run unit tests for the firebase function's data
|
536
|
-
--unit-test-python run unit tests for the python code
|
537
|
-
--no-unit-test-python
|
538
|
-
turn off run unit tests for the python code
|
539
529
|
```
|
540
530
|
</details>
|
541
531
|
|
@@ -0,0 +1,33 @@
|
|
1
|
+
runem/VERSION,sha256=gEtWWt0L1z5lxa4WmXDkvO_ucp0v_0QZPdvkbSU-bNs,7
|
2
|
+
runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
+
runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
|
4
|
+
runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
|
5
|
+
runem/blocking_print.py,sha256=S9dtgAeuTzc2-ht-vk9Wl6l-0PwS2tYbHDHDQQitrlA,841
|
6
|
+
runem/cli.py,sha256=wEt_Jnumhl8SiOdKdSJzLkJpWv6n3_Odhi_HeIixr1k,134
|
7
|
+
runem/command_line.py,sha256=1H1wZDfJrNCRly5yUBodYDvvKw8-4uLd3q-tcCvk1jM,11019
|
8
|
+
runem/config.py,sha256=y-e6j84FDiLSKKw9ShDzRlnS5t2e81MW8fKSKtxtJtg,5935
|
9
|
+
runem/config_metadata.py,sha256=Vy7dx8F-Z5jEp16OP2y6vHHoGkyhoCaTG4KIVkMWR7M,3232
|
10
|
+
runem/config_parse.py,sha256=6mCamzWu7HTotmqFJmLZg9FFE6qe1-rpmo8_v5ESPW8,13401
|
11
|
+
runem/files.py,sha256=CrwEiwPV0ZrrU3Ve0RWTTw9AtOQLekNXua347YYqcqQ,2748
|
12
|
+
runem/hook_manager.py,sha256=9T-4omyjBPZ6ua_37UWpT1dwNMbb4SKwvxYcN6fVxLE,4163
|
13
|
+
runem/informative_dict.py,sha256=U7p9z78UwOT4TAfng1iDXCEyeYz6C-XZlx9Z1pWNVrI,1548
|
14
|
+
runem/job.py,sha256=QVXvzz67fJk__-h0womFQsB80-w41E3XRcHpxmRnv3o,2912
|
15
|
+
runem/job_execute.py,sha256=xmH-O0qnsS2P_s3NCpXmla_kjsGgGCy0QdFH4pbY8iI,4000
|
16
|
+
runem/job_filter.py,sha256=fuxyKCHpTB4HlT_QagBk-IhhmWMlOr9Y9s5voP4yzYU,5370
|
17
|
+
runem/job_runner_simple_command.py,sha256=jxBukPm9bTLNhfQCkqNG5VepvB2ysmWAZwhBPHoTA6o,1091
|
18
|
+
runem/job_wrapper.py,sha256=wwPeWkMqhORl_Po6V-ofYZ-v9Ho0cFMprLRpY-30DyQ,964
|
19
|
+
runem/job_wrapper_python.py,sha256=m5xbWQxkDRtawjCcgxctzouv_Pny6bKiG2OPVE1hlgo,4226
|
20
|
+
runem/log.py,sha256=dIrocigvIJs1ZGkAzTogXkAK-0ZW3q5FkjpDgLdeW-E,630
|
21
|
+
runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
|
+
runem/report.py,sha256=IBCtMgGwnvVbEEqDWbYBGUZmTIzBLnpXqYSK5uu3vgk,8987
|
23
|
+
runem/run_command.py,sha256=Egl_j4bJ9mwi2JEFCsl0W6WH2IRgIdpMN7qdj8voClQ,6386
|
24
|
+
runem/runem.py,sha256=RIKF-l_ziGs0oKEueVkfygmnc_xiIdQo2qNDpiA-2Zs,13013
|
25
|
+
runem/runem_version.py,sha256=MbETwZO2Tb1Y3hX_OYZjKepEMKA1cjNvr-7Cqhz6e3s,271
|
26
|
+
runem/types.py,sha256=TLagRdB6-4gKqETAeJzo7-HFwBqQWGTwHcw2slSKN0U,7445
|
27
|
+
runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
|
28
|
+
runem-0.0.30.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
29
|
+
runem-0.0.30.dist-info/METADATA,sha256=HVftDpbRdnyeVk3Xi1c3VtWIcyVJH_2CUCkJy4qoKFs,29789
|
30
|
+
runem-0.0.30.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
31
|
+
runem-0.0.30.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
|
32
|
+
runem-0.0.30.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
|
33
|
+
runem-0.0.30.dist-info/RECORD,,
|