runem 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runem/VERSION +1 -1
- runem/command_line.py +3 -1
- runem/config.py +6 -1
- runem/config_metadata.py +4 -10
- runem/config_parse.py +5 -10
- runem/files.py +1 -1
- runem/hook_manager.py +5 -9
- runem/job.py +3 -1
- runem/job_execute.py +28 -17
- runem/job_filter.py +2 -8
- runem/job_runner_simple_command.py +1 -1
- runem/job_wrapper.py +2 -1
- runem/job_wrapper_python.py +3 -1
- runem/report.py +2 -3
- runem/runem.py +22 -48
- runem/types/__init__.py +12 -0
- runem/types/common.py +12 -0
- runem/types/errors.py +4 -0
- runem/types/filters.py +12 -0
- runem/types/hooks.py +15 -0
- runem/types/options.py +7 -0
- runem/types/runem_config.py +164 -0
- runem/types/types_jobs.py +156 -0
- {runem-0.1.0.dist-info → runem-0.1.2.dist-info}/METADATA +1 -1
- runem-0.1.2.dist-info/RECORD +42 -0
- {runem-0.1.0.dist-info → runem-0.1.2.dist-info}/top_level.txt +1 -0
- tests/test_types/__init__.py +0 -0
- tests/test_types/test_public_api.py +3 -0
- runem/types.py +0 -357
- runem-0.1.0.dist-info/RECORD +0 -33
- {runem-0.1.0.dist-info → runem-0.1.2.dist-info}/LICENSE +0 -0
- {runem-0.1.0.dist-info → runem-0.1.2.dist-info}/WHEEL +0 -0
- {runem-0.1.0.dist-info → runem-0.1.2.dist-info}/entry_points.txt +0 -0
runem/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.2
|
runem/command_line.py
CHANGED
@@ -8,7 +8,9 @@ from runem.config_metadata import ConfigMetadata
|
|
8
8
|
from runem.informative_dict import InformativeDict
|
9
9
|
from runem.log import error, log
|
10
10
|
from runem.runem_version import get_runem_version
|
11
|
-
from runem.types import JobNames
|
11
|
+
from runem.types.common import JobNames
|
12
|
+
from runem.types.options import OptionsWritable
|
13
|
+
from runem.types.runem_config import OptionConfig
|
12
14
|
from runem.utils import printable_set
|
13
15
|
|
14
16
|
|
runem/config.py
CHANGED
@@ -7,7 +7,12 @@ from packaging.version import Version
|
|
7
7
|
|
8
8
|
from runem.log import error, log
|
9
9
|
from runem.runem_version import get_runem_version
|
10
|
-
from runem.types import
|
10
|
+
from runem.types.runem_config import (
|
11
|
+
Config,
|
12
|
+
GlobalConfig,
|
13
|
+
GlobalSerialisedConfig,
|
14
|
+
UserConfigMetadata,
|
15
|
+
)
|
11
16
|
|
12
17
|
CFG_FILE_YAML = pathlib.Path(".runem.yml")
|
13
18
|
|
runem/config_metadata.py
CHANGED
@@ -3,16 +3,10 @@ import pathlib
|
|
3
3
|
import typing
|
4
4
|
|
5
5
|
from runem.informative_dict import InformativeDict
|
6
|
-
from runem.types import
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
OptionConfigs,
|
11
|
-
OptionsWritable,
|
12
|
-
OrderedPhases,
|
13
|
-
PhaseGroupedJobs,
|
14
|
-
TagFileFilters,
|
15
|
-
)
|
6
|
+
from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases
|
7
|
+
from runem.types.filters import TagFileFilters
|
8
|
+
from runem.types.options import OptionsWritable
|
9
|
+
from runem.types.runem_config import OptionConfigs, PhaseGroupedJobs
|
16
10
|
|
17
11
|
if typing.TYPE_CHECKING: # pragma: no cover
|
18
12
|
from runem.hook_manager import HookManager
|
runem/config_parse.py
CHANGED
@@ -10,28 +10,23 @@ from runem.hook_manager import HookManager
|
|
10
10
|
from runem.job import Job
|
11
11
|
from runem.job_wrapper import get_job_wrapper
|
12
12
|
from runem.log import error, log, warn
|
13
|
-
from runem.types import
|
13
|
+
from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases, PhaseName
|
14
|
+
from runem.types.errors import FunctionNotFound
|
15
|
+
from runem.types.filters import TagFileFilter, TagFileFilters
|
16
|
+
from runem.types.hooks import HookName
|
17
|
+
from runem.types.runem_config import (
|
14
18
|
Config,
|
15
19
|
ConfigNodes,
|
16
|
-
FunctionNotFound,
|
17
20
|
GlobalConfig,
|
18
21
|
GlobalSerialisedConfig,
|
19
22
|
HookConfig,
|
20
|
-
HookName,
|
21
23
|
Hooks,
|
22
24
|
HookSerialisedConfig,
|
23
25
|
JobConfig,
|
24
|
-
JobNames,
|
25
|
-
JobPhases,
|
26
26
|
JobSerialisedConfig,
|
27
|
-
JobTags,
|
28
27
|
JobWhen,
|
29
28
|
OptionConfigs,
|
30
|
-
OrderedPhases,
|
31
29
|
PhaseGroupedJobs,
|
32
|
-
PhaseName,
|
33
|
-
TagFileFilter,
|
34
|
-
TagFileFilters,
|
35
30
|
TagFileFilterSerialised,
|
36
31
|
)
|
37
32
|
|
runem/files.py
CHANGED
@@ -5,7 +5,7 @@ from pathlib import Path
|
|
5
5
|
from subprocess import check_output as subprocess_check_output
|
6
6
|
|
7
7
|
from runem.config_metadata import ConfigMetadata
|
8
|
-
from runem.types import FilePathListLookup
|
8
|
+
from runem.types.filters import FilePathListLookup
|
9
9
|
|
10
10
|
|
11
11
|
def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:
|
runem/hook_manager.py
CHANGED
@@ -7,15 +7,10 @@ from runem.config_metadata import ConfigMetadata
|
|
7
7
|
from runem.job import Job
|
8
8
|
from runem.job_execute import job_execute
|
9
9
|
from runem.log import log
|
10
|
-
from runem.types import
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
Hooks,
|
15
|
-
HookSpecificKwargs,
|
16
|
-
HooksStore,
|
17
|
-
JobConfig,
|
18
|
-
)
|
10
|
+
from runem.types.filters import FilePathListLookup
|
11
|
+
from runem.types.hooks import HookName
|
12
|
+
from runem.types.runem_config import HookConfig, Hooks, HooksStore, JobConfig
|
13
|
+
from runem.types.types_jobs import HookSpecificKwargs
|
19
14
|
|
20
15
|
|
21
16
|
class HookManager:
|
@@ -95,6 +90,7 @@ class HookManager:
|
|
95
90
|
job_execute(
|
96
91
|
job_config,
|
97
92
|
running_jobs={},
|
93
|
+
completed_jobs={},
|
98
94
|
config_metadata=config_metadata,
|
99
95
|
file_lists=file_lists,
|
100
96
|
**kwargs,
|
runem/job.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
import typing
|
2
2
|
|
3
|
-
from runem.types import FilePathList,
|
3
|
+
from runem.types.common import FilePathList, JobTags
|
4
|
+
from runem.types.filters import FilePathListLookup
|
5
|
+
from runem.types.runem_config import JobConfig, JobWhen
|
4
6
|
|
5
7
|
|
6
8
|
class NoJobName(ValueError):
|
runem/job_execute.py
CHANGED
@@ -12,14 +12,15 @@ from runem.informative_dict import ReadOnlyInformativeDict
|
|
12
12
|
from runem.job import Job
|
13
13
|
from runem.job_wrapper import get_job_wrapper
|
14
14
|
from runem.log import error, log
|
15
|
-
from runem.types import
|
16
|
-
|
17
|
-
|
15
|
+
from runem.types.common import FilePathList, JobTags
|
16
|
+
from runem.types.filters import FilePathListLookup
|
17
|
+
from runem.types.runem_config import JobConfig
|
18
|
+
from runem.types.types_jobs import (
|
19
|
+
AllKwargs,
|
18
20
|
HookSpecificKwargs,
|
19
|
-
JobConfig,
|
20
21
|
JobFunction,
|
22
|
+
JobKwargs,
|
21
23
|
JobReturn,
|
22
|
-
JobTags,
|
23
24
|
JobTiming,
|
24
25
|
TimingEntries,
|
25
26
|
TimingEntry,
|
@@ -80,19 +81,27 @@ def job_execute_inner(
|
|
80
81
|
log(f"job: running: '{Job.get_job_name(job_config)}'")
|
81
82
|
reports: JobReturn
|
82
83
|
try:
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
file_list
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
84
|
+
# Define the common args for all jobs and hooks.
|
85
|
+
job_k_args: JobKwargs = {
|
86
|
+
"config_metadata": config_metadata,
|
87
|
+
"file_list": file_list,
|
88
|
+
"job": job_config,
|
89
|
+
"label": Job.get_job_name(job_config),
|
90
|
+
"options": ReadOnlyInformativeDict(config_metadata.options),
|
91
|
+
"procs": config_metadata.args.procs,
|
92
|
+
"record_sub_job_time": _record_sub_job_time,
|
93
|
+
"root_path": root_path,
|
94
|
+
"verbose": config_metadata.args.verbose,
|
95
|
+
}
|
96
|
+
# Merge in the hook-specific kwargs (if any) for situations where we are
|
97
|
+
# calling hooks.
|
98
|
+
all_k_args: AllKwargs = {
|
99
|
+
**job_k_args,
|
94
100
|
**kwargs,
|
95
|
-
|
101
|
+
}
|
102
|
+
|
103
|
+
assert isinstance(function, JobFunction)
|
104
|
+
reports = function(**all_k_args)
|
96
105
|
except BaseException: # pylint: disable=broad-exception-caught
|
97
106
|
# log that we hit an error on this job and re-raise
|
98
107
|
log(decorate=False)
|
@@ -111,6 +120,7 @@ def job_execute_inner(
|
|
111
120
|
def job_execute(
|
112
121
|
job_config: JobConfig,
|
113
122
|
running_jobs: typing.Dict[str, str],
|
123
|
+
completed_jobs: typing.Dict[str, str],
|
114
124
|
config_metadata: ConfigMetadata,
|
115
125
|
file_lists: FilePathListLookup,
|
116
126
|
**kwargs: Unpack[HookSpecificKwargs],
|
@@ -127,5 +137,6 @@ def job_execute(
|
|
127
137
|
file_lists,
|
128
138
|
**kwargs,
|
129
139
|
)
|
140
|
+
completed_jobs[this_id] = running_jobs[this_id]
|
130
141
|
del running_jobs[this_id]
|
131
142
|
return results
|
runem/job_filter.py
CHANGED
@@ -4,14 +4,8 @@ from collections import defaultdict
|
|
4
4
|
from runem.config_metadata import ConfigMetadata
|
5
5
|
from runem.job import Job
|
6
6
|
from runem.log import log
|
7
|
-
from runem.types import
|
8
|
-
|
9
|
-
JobNames,
|
10
|
-
JobPhases,
|
11
|
-
JobTags,
|
12
|
-
PhaseGroupedJobs,
|
13
|
-
PhaseName,
|
14
|
-
)
|
7
|
+
from runem.types.common import JobNames, JobPhases, JobTags, PhaseName
|
8
|
+
from runem.types.runem_config import JobConfig, PhaseGroupedJobs
|
15
9
|
from runem.utils import printable_set
|
16
10
|
|
17
11
|
|
runem/job_wrapper.py
CHANGED
@@ -5,7 +5,8 @@ from runem.job_runner_simple_command import (
|
|
5
5
|
validate_simple_command,
|
6
6
|
)
|
7
7
|
from runem.job_wrapper_python import get_job_wrapper_py_func
|
8
|
-
from runem.types import
|
8
|
+
from runem.types.runem_config import JobWrapper
|
9
|
+
from runem.types.types_jobs import JobFunction
|
9
10
|
|
10
11
|
|
11
12
|
def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobFunction:
|
runem/job_wrapper_python.py
CHANGED
@@ -3,7 +3,9 @@ import sys
|
|
3
3
|
from importlib.util import module_from_spec
|
4
4
|
from importlib.util import spec_from_file_location as module_spec_from_file_location
|
5
5
|
|
6
|
-
from runem.types import FunctionNotFound
|
6
|
+
from runem.types.errors import FunctionNotFound
|
7
|
+
from runem.types.runem_config import JobWrapper
|
8
|
+
from runem.types.types_jobs import JobFunction
|
7
9
|
|
8
10
|
|
9
11
|
def _load_python_function_from_module(
|
runem/report.py
CHANGED
@@ -4,14 +4,13 @@ from collections import defaultdict
|
|
4
4
|
from datetime import timedelta
|
5
5
|
|
6
6
|
from runem.log import log
|
7
|
-
from runem.types import
|
7
|
+
from runem.types.common import OrderedPhases, PhaseName
|
8
|
+
from runem.types.types_jobs import (
|
8
9
|
JobReturn,
|
9
10
|
JobRunMetadatasByPhase,
|
10
11
|
JobRunReportByPhase,
|
11
12
|
JobRunTimesByPhase,
|
12
13
|
JobTiming,
|
13
|
-
OrderedPhases,
|
14
|
-
PhaseName,
|
15
14
|
ReportUrlInfo,
|
16
15
|
ReportUrls,
|
17
16
|
TimingEntries,
|
runem/runem.py
CHANGED
@@ -28,7 +28,7 @@ import typing
|
|
28
28
|
from collections import defaultdict
|
29
29
|
from datetime import timedelta
|
30
30
|
from itertools import repeat
|
31
|
-
from multiprocessing.managers import DictProxy,
|
31
|
+
from multiprocessing.managers import DictProxy, ValueProxy
|
32
32
|
from timeit import default_timer as timer
|
33
33
|
from types import TracebackType
|
34
34
|
|
@@ -41,23 +41,19 @@ from runem.config import load_project_config, load_user_configs
|
|
41
41
|
from runem.config_metadata import ConfigMetadata
|
42
42
|
from runem.config_parse import load_config_metadata
|
43
43
|
from runem.files import find_files
|
44
|
-
from runem.job import Job
|
45
44
|
from runem.job_execute import job_execute
|
46
45
|
from runem.job_filter import filter_jobs
|
47
46
|
from runem.log import error, log, warn
|
48
47
|
from runem.report import report_on_run
|
49
|
-
from runem.types import
|
50
|
-
|
51
|
-
|
52
|
-
|
48
|
+
from runem.types.common import OrderedPhases, PhaseName
|
49
|
+
from runem.types.filters import FilePathListLookup
|
50
|
+
from runem.types.hooks import HookName
|
51
|
+
from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
|
52
|
+
from runem.types.types_jobs import (
|
53
53
|
JobReturn,
|
54
54
|
JobRunMetadata,
|
55
55
|
JobRunMetadatasByPhase,
|
56
|
-
Jobs,
|
57
56
|
JobTiming,
|
58
|
-
OrderedPhases,
|
59
|
-
PhaseGroupedJobs,
|
60
|
-
PhaseName,
|
61
57
|
)
|
62
58
|
from runem.utils import printable_set
|
63
59
|
|
@@ -123,7 +119,7 @@ class DummySpinner(ConsoleRenderable): # pragma: no cover
|
|
123
119
|
def _update_progress(
|
124
120
|
label: str,
|
125
121
|
running_jobs: typing.Dict[str, str],
|
126
|
-
|
122
|
+
completed_jobs: typing.Dict[str, str],
|
127
123
|
all_jobs: Jobs,
|
128
124
|
is_running: ValueProxy[bool],
|
129
125
|
num_workers: int,
|
@@ -134,7 +130,6 @@ def _update_progress(
|
|
134
130
|
Args:
|
135
131
|
label (str): The identifier.
|
136
132
|
running_jobs (Dict[str, str]): The currently running jobs.
|
137
|
-
seen_jobs (List[str]): Jobs that the function has previously tracked.
|
138
133
|
all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
|
139
134
|
is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
|
140
135
|
num_workers (int): Indicates the number of workers performing the jobs.
|
@@ -146,47 +141,25 @@ def _update_progress(
|
|
146
141
|
else:
|
147
142
|
spinner = DummySpinner()
|
148
143
|
|
149
|
-
|
150
|
-
|
151
|
-
# The set of all job labels, and the set of completed jobs
|
152
|
-
all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
|
153
|
-
completed_jobs: typing.Set[str] = set()
|
154
|
-
|
155
|
-
# This dataset is used to track changes between iterations
|
156
|
-
last_running_jobs_set: typing.Set[str] = set()
|
144
|
+
last_running_jobs_set: typing.Set[str] = set()
|
157
145
|
|
146
|
+
with rich_console.status(spinner):
|
158
147
|
while is_running.value:
|
159
148
|
running_jobs_set: typing.Set[str] = set(running_jobs.values())
|
160
|
-
seen_jobs = list(running_jobs_set.union(seen_jobs)) # Update the seen jobs
|
161
|
-
|
162
|
-
# Jobs that have disappeared since last check
|
163
|
-
disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
|
164
|
-
|
165
|
-
# Jobs that have not yet completed
|
166
|
-
remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
|
167
149
|
|
168
|
-
#
|
169
|
-
workers_retiring: bool = len(remaining_jobs) <= num_workers
|
170
|
-
|
171
|
-
if workers_retiring:
|
172
|
-
# Handle edge case: a task may have disappeared whilst process was sleeping
|
173
|
-
all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
|
174
|
-
disappeared_jobs.update(all_completed_jobs - running_jobs_set)
|
175
|
-
|
176
|
-
completed_jobs.update(disappeared_jobs)
|
177
|
-
|
178
|
-
# Prepare progress report
|
150
|
+
# Progress report
|
179
151
|
progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
|
180
|
-
running_jobs_list = printable_set(
|
152
|
+
running_jobs_list = printable_set(
|
153
|
+
running_jobs_set
|
154
|
+
) # Reflect current running jobs accurately
|
155
|
+
report: str = f"{label}: {progress}({num_workers}): {running_jobs_list}"
|
181
156
|
if show_spinner:
|
182
|
-
spinner.text =
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
# Update the tracked dataset for the next iteration
|
187
|
-
last_running_jobs_set = running_jobs_set
|
157
|
+
spinner.text = report
|
158
|
+
else:
|
159
|
+
if last_running_jobs_set != running_jobs_set:
|
160
|
+
rich_console.log(report)
|
188
161
|
|
189
|
-
# Sleep
|
162
|
+
# Sleep for reduced CPU usage
|
190
163
|
time.sleep(0.1)
|
191
164
|
|
192
165
|
|
@@ -225,8 +198,8 @@ def _process_jobs(
|
|
225
198
|
subprocess_error: typing.Optional[BaseException] = None
|
226
199
|
|
227
200
|
with multiprocessing.Manager() as manager:
|
228
|
-
seen_jobs: ListProxy[str] = manager.list()
|
229
201
|
running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
|
202
|
+
completed_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
|
230
203
|
is_running: ValueProxy[bool] = manager.Value("b", True)
|
231
204
|
|
232
205
|
terminal_writer_process = multiprocessing.Process(
|
@@ -234,7 +207,7 @@ def _process_jobs(
|
|
234
207
|
args=(
|
235
208
|
phase,
|
236
209
|
running_jobs,
|
237
|
-
|
210
|
+
completed_jobs,
|
238
211
|
jobs,
|
239
212
|
is_running,
|
240
213
|
num_concurrent_procs,
|
@@ -251,6 +224,7 @@ def _process_jobs(
|
|
251
224
|
zip(
|
252
225
|
jobs,
|
253
226
|
repeat(running_jobs),
|
227
|
+
repeat(completed_jobs),
|
254
228
|
repeat(config_metadata),
|
255
229
|
repeat(file_lists),
|
256
230
|
),
|
runem/types/__init__.py
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
from runem.types.common import FilePathList, JobName
|
2
|
+
from runem.types.options import Options
|
3
|
+
from runem.types.types_jobs import HookKwargs, JobKwargs, JobReturnData
|
4
|
+
|
5
|
+
__all__ = [
|
6
|
+
"FilePathList",
|
7
|
+
"HookKwargs",
|
8
|
+
"JobName",
|
9
|
+
"JobReturnData",
|
10
|
+
"Options",
|
11
|
+
"JobKwargs",
|
12
|
+
]
|
runem/types/common.py
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
import typing
|
2
|
+
|
3
|
+
# meta-data types
|
4
|
+
JobName = str
|
5
|
+
JobTag = str
|
6
|
+
JobNames = typing.Set[JobName]
|
7
|
+
JobPhases = typing.Set[str]
|
8
|
+
JobTags = typing.Set[JobTag]
|
9
|
+
PhaseName = str
|
10
|
+
OrderedPhases = typing.Tuple[PhaseName, ...]
|
11
|
+
FilePathSerialise = str
|
12
|
+
FilePathList = typing.List[FilePathSerialise]
|
runem/types/errors.py
ADDED
runem/types/filters.py
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
import typing
|
2
|
+
|
3
|
+
from runem.types.common import FilePathList, JobTag
|
4
|
+
|
5
|
+
|
6
|
+
class TagFileFilter(typing.TypedDict):
|
7
|
+
tag: JobTag
|
8
|
+
regex: str
|
9
|
+
|
10
|
+
|
11
|
+
TagFileFilters = typing.Dict[JobTag, TagFileFilter]
|
12
|
+
FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
|
runem/types/hooks.py
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
import enum
|
2
|
+
|
3
|
+
|
4
|
+
class HookName(enum.Enum):
|
5
|
+
"""List supported hooks.
|
6
|
+
|
7
|
+
TODO:
|
8
|
+
- before all tasks are run, after config is read
|
9
|
+
- BEFORE_ALL = "before-all"
|
10
|
+
- after all tasks are done, before reporting
|
11
|
+
- AFTER_ALL = "after-all"
|
12
|
+
"""
|
13
|
+
|
14
|
+
# at exit
|
15
|
+
ON_EXIT = "on-exit"
|
runem/types/options.py
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
|
2
|
+
|
3
|
+
OptionName = str
|
4
|
+
OptionValue = bool
|
5
|
+
OptionsWritable = InformativeDict[OptionName, OptionValue]
|
6
|
+
OptionsReadOnly = ReadOnlyInformativeDict[OptionName, OptionValue]
|
7
|
+
Options = OptionsReadOnly
|
@@ -0,0 +1,164 @@
|
|
1
|
+
import pathlib
|
2
|
+
import typing
|
3
|
+
|
4
|
+
from runem.types.common import JobName, JobTags, OrderedPhases, PhaseName
|
5
|
+
from runem.types.filters import TagFileFilter
|
6
|
+
from runem.types.hooks import HookName
|
7
|
+
|
8
|
+
|
9
|
+
class OptionConfig(typing.TypedDict, total=False):
|
10
|
+
"""Spec for configuring job option overrides."""
|
11
|
+
|
12
|
+
name: str
|
13
|
+
aliases: typing.Optional[typing.List[str]]
|
14
|
+
alias: typing.Optional[str]
|
15
|
+
default: bool
|
16
|
+
type: str
|
17
|
+
desc: typing.Optional[str]
|
18
|
+
|
19
|
+
|
20
|
+
OptionConfigs = typing.Tuple[OptionConfig, ...]
|
21
|
+
|
22
|
+
|
23
|
+
class OptionConfigSerialised(typing.TypedDict):
|
24
|
+
"""Supports better serialisation of options."""
|
25
|
+
|
26
|
+
option: OptionConfig
|
27
|
+
|
28
|
+
|
29
|
+
class JobParamConfig(typing.TypedDict):
|
30
|
+
"""Configures what parameters are passed to the test-callable.
|
31
|
+
|
32
|
+
FIXME: this isn't actually used at all, yet
|
33
|
+
"""
|
34
|
+
|
35
|
+
limitFilesToGroup: bool # whether to limit file-set for the job
|
36
|
+
|
37
|
+
|
38
|
+
class JobAddressConfig(typing.TypedDict):
|
39
|
+
"""Configuration which described a callable to call."""
|
40
|
+
|
41
|
+
file: str # the file-module where 'function' can be found
|
42
|
+
function: str # the 'function' in module to run
|
43
|
+
|
44
|
+
|
45
|
+
class JobContextConfig(typing.TypedDict, total=False):
|
46
|
+
# what parameters the job needs # DEFUNCT
|
47
|
+
params: typing.Optional[JobParamConfig]
|
48
|
+
|
49
|
+
# the path or paths to run the command in. If given a list the job will be
|
50
|
+
# duplicated for each given path.
|
51
|
+
cwd: typing.Optional[typing.Union[str, typing.List[str]]]
|
52
|
+
|
53
|
+
|
54
|
+
class JobWhen(typing.TypedDict, total=False):
|
55
|
+
"""Configures WHEN to call the callable i.e. priority."""
|
56
|
+
|
57
|
+
tags: JobTags # the job tags - used for filtering job-types
|
58
|
+
phase: PhaseName # the phase when the job should be run
|
59
|
+
|
60
|
+
|
61
|
+
class JobWrapper(typing.TypedDict, total=False):
|
62
|
+
"""A base-type for jobs, hooks, and things that can be invoked."""
|
63
|
+
|
64
|
+
addr: JobAddressConfig # which callable to call
|
65
|
+
command: str # a one-liner command to be run
|
66
|
+
|
67
|
+
|
68
|
+
class JobConfig(JobWrapper, total=False):
|
69
|
+
"""A dict that defines a job to be run.
|
70
|
+
|
71
|
+
It consists of the label, address, context and filter information
|
72
|
+
|
73
|
+
TODO: make a class variant of this
|
74
|
+
"""
|
75
|
+
|
76
|
+
label: JobName # the name of the job
|
77
|
+
ctx: typing.Optional[JobContextConfig] # how to call the callable
|
78
|
+
when: JobWhen # when to call the job
|
79
|
+
|
80
|
+
|
81
|
+
Jobs = typing.List[JobConfig]
|
82
|
+
|
83
|
+
|
84
|
+
class TagFileFilterSerialised(typing.TypedDict):
|
85
|
+
"""Supports better serialisation of TagFileFilters."""
|
86
|
+
|
87
|
+
filter: TagFileFilter
|
88
|
+
|
89
|
+
|
90
|
+
class GlobalConfig(typing.TypedDict):
|
91
|
+
"""The config for the entire test run."""
|
92
|
+
|
93
|
+
# Phases control the order of jobs, jobs earlier in the stack get run earlier
|
94
|
+
# the core ide here is to ensure that certain types of job-dependencies,
|
95
|
+
# such as code-reformatting jobs run before analysis tools, therefore making
|
96
|
+
# any error messages about the code give consistent line numbers e..g if a
|
97
|
+
# re-formatter edits a file the error line will move and the analysis phase
|
98
|
+
# will report the wrong line.
|
99
|
+
phases: OrderedPhases
|
100
|
+
|
101
|
+
# Options control the extra flags that are optionally consumed by job.
|
102
|
+
# Options configured here are used to set command-line-options. All options
|
103
|
+
# and their current state are passed to each job.
|
104
|
+
options: typing.Optional[typing.List[OptionConfigSerialised]]
|
105
|
+
|
106
|
+
# File filters control which files will be passed to jobs for a given tags.
|
107
|
+
# Job will receive the super-set of files for all that job's tags.
|
108
|
+
files: typing.Optional[typing.List[TagFileFilterSerialised]]
|
109
|
+
|
110
|
+
# Which minimal version of runem does this config support?
|
111
|
+
min_version: typing.Optional[str]
|
112
|
+
|
113
|
+
|
114
|
+
class GlobalSerialisedConfig(typing.TypedDict):
|
115
|
+
"""Intended to make reading a config file easier.
|
116
|
+
|
117
|
+
Unlike JobSerialisedConfig, this type may not actually help readability.
|
118
|
+
|
119
|
+
An intermediary type for serialisation of the global config, the 'global' resides
|
120
|
+
inside a 'global' key and therefore is easier to find and reason about.
|
121
|
+
"""
|
122
|
+
|
123
|
+
config: GlobalConfig
|
124
|
+
|
125
|
+
|
126
|
+
class HookConfig(JobWrapper, total=False):
|
127
|
+
"""Specification for hooks.
|
128
|
+
|
129
|
+
Like JobConfig with use addr or command to specify what to execute.
|
130
|
+
"""
|
131
|
+
|
132
|
+
hook_name: HookName # the hook for when this is called
|
133
|
+
|
134
|
+
|
135
|
+
class HookSerialisedConfig(typing.TypedDict):
|
136
|
+
"""Intended to make reading a config file easier.
|
137
|
+
|
138
|
+
Also, unlike JobSerialisedConfig, this type may not actually help readability.
|
139
|
+
"""
|
140
|
+
|
141
|
+
hook: HookConfig
|
142
|
+
|
143
|
+
|
144
|
+
class JobSerialisedConfig(typing.TypedDict):
|
145
|
+
"""Makes serialised configs easier to read.
|
146
|
+
|
147
|
+
An intermediary typ for serialisation as each 'job' resides inside a 'job' key.
|
148
|
+
|
149
|
+
This makes formatting of YAML config _significantly_ easier to understand.
|
150
|
+
"""
|
151
|
+
|
152
|
+
job: JobConfig
|
153
|
+
|
154
|
+
|
155
|
+
ConfigNodes = typing.Union[
|
156
|
+
GlobalSerialisedConfig, JobSerialisedConfig, HookSerialisedConfig
|
157
|
+
]
|
158
|
+
# The config format as it is serialised to/from disk
|
159
|
+
Config = typing.List[ConfigNodes]
|
160
|
+
UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
|
161
|
+
Hooks = typing.DefaultDict[HookName, typing.List[HookConfig]]
|
162
|
+
# A dictionary to hold hooks, with hook names as keys
|
163
|
+
HooksStore = typing.Dict[HookName, typing.List[HookConfig]]
|
164
|
+
PhaseGroupedJobs = typing.DefaultDict[PhaseName, Jobs]
|
@@ -0,0 +1,156 @@
|
|
1
|
+
"""
|
2
|
+
|
3
|
+
Some note on Unpack and kwargs:
|
4
|
+
We *try* to strongly type `**kwargs` for clarity.
|
5
|
+
We have tried several ways to define a Generic type that encapsulates
|
6
|
+
`**kwargs: SingleType`
|
7
|
+
... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
|
8
|
+
so we have to recommend instead using:
|
9
|
+
`**kwargs: Unpack[KwArgsType]`
|
10
|
+
|
11
|
+
For this to work across versions of python where support for Unpack changes;
|
12
|
+
for example `Unpack` is a python 3.12 feature, but available in the
|
13
|
+
`typing_extensions` module.
|
14
|
+
|
15
|
+
So, for now, it looks like we get away with importing `Unpack` from the
|
16
|
+
`typing_extensions` module, even in python 3.12, so we will use, and
|
17
|
+
recommend using, the `typing_extensions` of `Unpack`, until it becomes
|
18
|
+
obsolete.
|
19
|
+
|
20
|
+
Alternatively, we can use the following, but it's unnecessarily verbose.
|
21
|
+
|
22
|
+
if sys.version_info >= (3, 12): # pragma: no coverage
|
23
|
+
from typing import Unpack
|
24
|
+
else: # pragma: no coverage
|
25
|
+
from typing_extensions import Unpack
|
26
|
+
"""
|
27
|
+
|
28
|
+
import pathlib
|
29
|
+
import typing
|
30
|
+
from datetime import timedelta
|
31
|
+
|
32
|
+
from typing_extensions import Unpack
|
33
|
+
|
34
|
+
from runem.types.common import FilePathList, PhaseName
|
35
|
+
from runem.types.options import Options
|
36
|
+
from runem.types.runem_config import JobConfig
|
37
|
+
|
38
|
+
if typing.TYPE_CHECKING: # pragma: no cover
|
39
|
+
from runem.config_metadata import ConfigMetadata
|
40
|
+
|
41
|
+
ReportName = str
|
42
|
+
ReportUrl = typing.Union[str, pathlib.Path]
|
43
|
+
ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
|
44
|
+
ReportUrls = typing.List[ReportUrlInfo]
|
45
|
+
|
46
|
+
|
47
|
+
class JobReturnData(typing.TypedDict, total=False):
|
48
|
+
"""A dict that defines job result to be reported to the user."""
|
49
|
+
|
50
|
+
reportUrls: ReportUrls # urls containing reports for the user
|
51
|
+
|
52
|
+
|
53
|
+
TimingEntry = typing.Tuple[str, timedelta]
|
54
|
+
TimingEntries = typing.List[TimingEntry]
|
55
|
+
|
56
|
+
|
57
|
+
class JobTiming(typing.TypedDict, total=True):
|
58
|
+
"""A hierarchy of timing info. Job->JobCommands.
|
59
|
+
|
60
|
+
The overall time for a job is in 'job', the child calls to run_command are in
|
61
|
+
'commands'
|
62
|
+
"""
|
63
|
+
|
64
|
+
job: TimingEntry # the overall time for a job
|
65
|
+
commands: TimingEntries # timing for each call to `run_command`
|
66
|
+
|
67
|
+
|
68
|
+
JobReturn = typing.Optional[JobReturnData]
|
69
|
+
JobRunMetadata = typing.Tuple[JobTiming, JobReturn]
|
70
|
+
JobRunTimesByPhase = typing.Dict[PhaseName, typing.List[JobTiming]]
|
71
|
+
JobRunReportByPhase = typing.Dict[PhaseName, ReportUrls]
|
72
|
+
JobRunMetadatasByPhase = typing.Dict[PhaseName, typing.List[JobRunMetadata]]
|
73
|
+
|
74
|
+
|
75
|
+
class CommonKwargs(
|
76
|
+
typing.TypedDict,
|
77
|
+
total=True, # each of these are guaranteed to exist in jobs and hooks
|
78
|
+
):
|
79
|
+
"""Defines the base args that are passed to all jobs.
|
80
|
+
|
81
|
+
As we call hooks and job-task in the same manner, this defines the variables that we
|
82
|
+
can access from both hooks and job-tasks.
|
83
|
+
"""
|
84
|
+
|
85
|
+
config_metadata: "ConfigMetadata" # gives greater context to jobs and hooks
|
86
|
+
job: JobConfig # the job or hook task spec ¢ TODO: rename this
|
87
|
+
label: str # the name of the hook or the job-label
|
88
|
+
options: Options # options passed in on the command line
|
89
|
+
procs: int # the max number of concurrent procs to run
|
90
|
+
root_path: pathlib.Path # the path where the .runem.yml file is
|
91
|
+
verbose: bool # control log verbosity
|
92
|
+
|
93
|
+
|
94
|
+
class HookSpecificKwargs(typing.TypedDict, total=False):
|
95
|
+
"""Defines the args that are passed down to the hooks.
|
96
|
+
|
97
|
+
NOTE: that although these however
|
98
|
+
outside of the *hook* context, the data will not be present. Such is the
|
99
|
+
difficulty in dynamic programming.
|
100
|
+
"""
|
101
|
+
|
102
|
+
wall_clock_time_saved: timedelta # only on `HookName.ON_EXIT`
|
103
|
+
|
104
|
+
|
105
|
+
class JobTaskKwargs(
|
106
|
+
typing.TypedDict,
|
107
|
+
total=False, # for now, we don't enforce these types for job-context, but we should.
|
108
|
+
):
|
109
|
+
"""Defines the task-specific args for job-task functions."""
|
110
|
+
|
111
|
+
file_list: FilePathList
|
112
|
+
record_sub_job_time: typing.Optional[typing.Callable[[str, timedelta], None]]
|
113
|
+
|
114
|
+
|
115
|
+
class HookKwargs(CommonKwargs, HookSpecificKwargs):
|
116
|
+
"""A merged set of kwargs for runem-hooks."""
|
117
|
+
|
118
|
+
pass
|
119
|
+
|
120
|
+
|
121
|
+
class JobKwargs(CommonKwargs, JobTaskKwargs):
|
122
|
+
"""A merged set of kwargs for job-tasks."""
|
123
|
+
|
124
|
+
pass
|
125
|
+
|
126
|
+
|
127
|
+
class AllKwargs(CommonKwargs, JobTaskKwargs, HookSpecificKwargs):
|
128
|
+
"""A merged set of kwargs for al job-functions."""
|
129
|
+
|
130
|
+
pass
|
131
|
+
|
132
|
+
|
133
|
+
@typing.runtime_checkable
|
134
|
+
class JobFunction(typing.Protocol):
|
135
|
+
def __call__(self, **kwargs: Unpack[AllKwargs]) -> JobReturn: # pragma: no cover
|
136
|
+
"""Defines the call() protocol's abstract pattern for job-tasks."""
|
137
|
+
|
138
|
+
@property
|
139
|
+
def __name__(self) -> str: # pragma: no cover
|
140
|
+
"""Defines the name protocol for job-task functions.
|
141
|
+
|
142
|
+
This is primarily used for internal tests but can be useful for introspection.
|
143
|
+
"""
|
144
|
+
|
145
|
+
|
146
|
+
def _hook_example(
|
147
|
+
wall_clock_time_saved: timedelta,
|
148
|
+
**kwargs: typing.Any,
|
149
|
+
) -> None:
|
150
|
+
"""An example hook."""
|
151
|
+
|
152
|
+
|
153
|
+
def _job_task_example(
|
154
|
+
**kwargs: Unpack[JobKwargs],
|
155
|
+
) -> None:
|
156
|
+
"""An example job-task function."""
|
@@ -0,0 +1,42 @@
|
|
1
|
+
runem/VERSION,sha256=KXtQ8IOG01Ifj9ry94642dCs30UtLmFZaQX7o3IIx5I,6
|
2
|
+
runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
+
runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
|
4
|
+
runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
|
5
|
+
runem/blocking_print.py,sha256=S9dtgAeuTzc2-ht-vk9Wl6l-0PwS2tYbHDHDQQitrlA,841
|
6
|
+
runem/cli.py,sha256=wEt_Jnumhl8SiOdKdSJzLkJpWv6n3_Odhi_HeIixr1k,134
|
7
|
+
runem/command_line.py,sha256=HsZjxJsVQZLfAeSRI6cpx6P1foeuqCYhHmeQ8Xg9RHY,13774
|
8
|
+
runem/config.py,sha256=UiEU0Jyg5qjrNStvasWYjMOABQHhpZjbPiX3-sH_CMg,5969
|
9
|
+
runem/config_metadata.py,sha256=krDomUcADsAeUQrxwNmOS58eeaNIlqmhWIKWv8mUH4A,3300
|
10
|
+
runem/config_parse.py,sha256=sXPMA8HSUcaJoq0dUfphK181waZqnIrq20mpYOQ_XCo,13498
|
11
|
+
runem/files.py,sha256=59boeFvUANYOS-PllIjeKIht6lNINZ43WxahDg90oAc,4392
|
12
|
+
runem/hook_manager.py,sha256=H0TL3HCqU2mgKm_-dgCD7TsK5T1bLT4g7x6kpytMPhU,4350
|
13
|
+
runem/informative_dict.py,sha256=U7p9z78UwOT4TAfng1iDXCEyeYz6C-XZlx9Z1pWNVrI,1548
|
14
|
+
runem/job.py,sha256=LmNXDHxDxAwyJxAAdPHIv_0bwZy2btyTPHPGO_N7euI,2986
|
15
|
+
runem/job_execute.py,sha256=BPkeTpeTGJs3QWa0-07DZvF1f0uKO79e4yMsTxq1UHk,4656
|
16
|
+
runem/job_filter.py,sha256=7vgG4YWJ9gyGBFjV7QbSojG5ofYoszAmxXx9HnMLkHo,5384
|
17
|
+
runem/job_runner_simple_command.py,sha256=OIfj7CrPy0fPszhubUsTzORMlMC62OiNSvH5DysYL-4,1104
|
18
|
+
runem/job_wrapper.py,sha256=q5GtopZ5vhSJ581rwU4-lF9KnbL3ZYgOC8fqaCnXD_g,983
|
19
|
+
runem/job_wrapper_python.py,sha256=rx7J_N-JXs8GgMq7Sla7B9s_ZAfofKUhEnzgMcq_bts,4303
|
20
|
+
runem/log.py,sha256=dIrocigvIJs1ZGkAzTogXkAK-0ZW3q5FkjpDgLdeW-E,630
|
21
|
+
runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
|
+
runem/report.py,sha256=IedwW9mmJfGC6vKQdKDHQH5eoiYzTWHaaD4a3J4e_Pc,9020
|
23
|
+
runem/run_command.py,sha256=Egl_j4bJ9mwi2JEFCsl0W6WH2IRgIdpMN7qdj8voClQ,6386
|
24
|
+
runem/runem.py,sha256=QbJU48zx1hyiRIG5Kf4DAa4Kka2dOKYgqlZ1T6fN62k,13316
|
25
|
+
runem/runem_version.py,sha256=MbETwZO2Tb1Y3hX_OYZjKepEMKA1cjNvr-7Cqhz6e3s,271
|
26
|
+
runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
|
27
|
+
runem/types/__init__.py,sha256=6TzF4KV9tDGuDTr2qAXmWWkfDU52WuVlQ8Hcz48aYDk,286
|
28
|
+
runem/types/common.py,sha256=gPMSoJ3yRUYjHnoviRrpSg0gRwsGLFGWGpbTWkq4jX0,279
|
29
|
+
runem/types/errors.py,sha256=rbM5BA6UhY1X7Q0OZLUNsG7JXAjgNFTG5KQuqPNuZm8,103
|
30
|
+
runem/types/filters.py,sha256=8R5fyMssN0ISGBilJhEtbdHFl6OP7uI51WKkB5SH6EA,255
|
31
|
+
runem/types/hooks.py,sha256=lgrv5QAuHCEzr5dXDj4-azNcs63addY9zdrGWj5zv_s,292
|
32
|
+
runem/types/options.py,sha256=y8_hyWYvhalC9-kZbvoDtxm0trZgyyGcswQqfuQy_pM,265
|
33
|
+
runem/types/runem_config.py,sha256=qG_bghm5Nr-ZTbaZbf1v8Fx447V-hgEvvRy5NZ3t-Io,5141
|
34
|
+
runem/types/types_jobs.py,sha256=wqiiBmRIJDbGlKcfOqewHGKx350w0p4_7pysMm7xGmo,4906
|
35
|
+
tests/test_types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
36
|
+
tests/test_types/test_public_api.py,sha256=QHiwt7CetQur65JSbFRnOzQxhCJkX5MVLymHHVd_6yc,160
|
37
|
+
runem-0.1.2.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
38
|
+
runem-0.1.2.dist-info/METADATA,sha256=UQCyI88PE3lJWibR7VmDFyIGi9xPpnykhp8QKmn9E-U,5842
|
39
|
+
runem-0.1.2.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
40
|
+
runem-0.1.2.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
|
41
|
+
runem-0.1.2.dist-info/top_level.txt,sha256=Zu65aVeDPh8WbChU4Mi7-Md4S3XJDPuqdQjEE3DSQno,12
|
42
|
+
runem-0.1.2.dist-info/RECORD,,
|
File without changes
|
runem/types.py
DELETED
@@ -1,357 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
|
3
|
-
Some note on Unpack and kwargs:
|
4
|
-
We *try* to strongly type `**kwargs` for clarity.
|
5
|
-
We have tried several ways to define a Generic type that encapsulates
|
6
|
-
`**kwargs: SingleType`
|
7
|
-
... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
|
8
|
-
so we have to recommend instead using:
|
9
|
-
`**kwargs: Unpack[KwArgsType]`
|
10
|
-
|
11
|
-
For this to work across versions of python where support for Unpack changes;
|
12
|
-
for example `Unpack` is a python 3.12 feature, but available in the
|
13
|
-
`typing_extensions` module.
|
14
|
-
|
15
|
-
So, for now, it looks like we get away with importing `Unpack` from the
|
16
|
-
`typing_extensions` module, even in python 3.12, so we will use, and
|
17
|
-
recommend using, the `typing_extensions` of `Unpack`, until it becomes
|
18
|
-
obsolete.
|
19
|
-
|
20
|
-
Alternatively, we can use the following, but it's unnecessarily verbose.
|
21
|
-
|
22
|
-
if sys.version_info >= (3, 12): # pragma: no coverage
|
23
|
-
from typing import Unpack
|
24
|
-
else: # pragma: no coverage
|
25
|
-
from typing_extensions import Unpack
|
26
|
-
"""
|
27
|
-
|
28
|
-
import pathlib
|
29
|
-
import typing
|
30
|
-
from datetime import timedelta
|
31
|
-
from enum import Enum
|
32
|
-
|
33
|
-
from typing_extensions import Unpack
|
34
|
-
|
35
|
-
from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
|
36
|
-
|
37
|
-
|
38
|
-
class FunctionNotFound(ValueError):
|
39
|
-
"""Thrown when the test-function cannot be found."""
|
40
|
-
|
41
|
-
pass
|
42
|
-
|
43
|
-
|
44
|
-
# meta-data types
|
45
|
-
JobName = str
|
46
|
-
JobTag = str
|
47
|
-
JobNames = typing.Set[JobName]
|
48
|
-
JobPhases = typing.Set[str]
|
49
|
-
JobTags = typing.Set[JobTag]
|
50
|
-
PhaseName = str
|
51
|
-
OrderedPhases = typing.Tuple[PhaseName, ...]
|
52
|
-
ReportName = str
|
53
|
-
ReportUrl = typing.Union[str, pathlib.Path]
|
54
|
-
ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
|
55
|
-
ReportUrls = typing.List[ReportUrlInfo]
|
56
|
-
|
57
|
-
|
58
|
-
class HookName(Enum):
|
59
|
-
# at exit
|
60
|
-
ON_EXIT = "on-exit"
|
61
|
-
# before all tasks are run, after config is read
|
62
|
-
# BEFORE_ALL = "before-all"
|
63
|
-
# after all tasks are done, before reporting
|
64
|
-
# AFTER_ALL = "after-all"
|
65
|
-
|
66
|
-
|
67
|
-
class JobReturnData(typing.TypedDict, total=False):
|
68
|
-
"""A dict that defines job result to be reported to the user."""
|
69
|
-
|
70
|
-
reportUrls: ReportUrls # urls containing reports for the user
|
71
|
-
|
72
|
-
|
73
|
-
TimingEntry = typing.Tuple[str, timedelta]
|
74
|
-
TimingEntries = typing.List[TimingEntry]
|
75
|
-
|
76
|
-
|
77
|
-
class JobTiming(typing.TypedDict, total=True):
|
78
|
-
"""A hierarchy of timing info. Job->JobCommands.
|
79
|
-
|
80
|
-
The overall time for a job is in 'job', the child calls to run_command are in
|
81
|
-
'commands'
|
82
|
-
"""
|
83
|
-
|
84
|
-
job: TimingEntry # the overall time for a job
|
85
|
-
commands: TimingEntries # timing for each call to `run_command`
|
86
|
-
|
87
|
-
|
88
|
-
JobReturn = typing.Optional[JobReturnData]
|
89
|
-
JobRunMetadata = typing.Tuple[JobTiming, JobReturn]
|
90
|
-
JobRunTimesByPhase = typing.Dict[PhaseName, typing.List[JobTiming]]
|
91
|
-
JobRunReportByPhase = typing.Dict[PhaseName, ReportUrls]
|
92
|
-
JobRunMetadatasByPhase = typing.Dict[PhaseName, typing.List[JobRunMetadata]]
|
93
|
-
|
94
|
-
|
95
|
-
class OptionConfig(typing.TypedDict, total=False):
|
96
|
-
"""Spec for configuring job option overrides."""
|
97
|
-
|
98
|
-
name: str
|
99
|
-
aliases: typing.Optional[typing.List[str]]
|
100
|
-
alias: typing.Optional[str]
|
101
|
-
default: bool
|
102
|
-
type: str
|
103
|
-
desc: typing.Optional[str]
|
104
|
-
|
105
|
-
|
106
|
-
OptionName = str
|
107
|
-
OptionValue = bool
|
108
|
-
|
109
|
-
OptionConfigs = typing.Tuple[OptionConfig, ...]
|
110
|
-
OptionsWritable = InformativeDict[OptionName, OptionValue]
|
111
|
-
OptionsReadOnly = ReadOnlyInformativeDict[OptionName, OptionValue]
|
112
|
-
Options = OptionsReadOnly
|
113
|
-
|
114
|
-
# P1: bool for verbose, P2: list of file paths to work on
|
115
|
-
|
116
|
-
|
117
|
-
class TagFileFilter(typing.TypedDict):
|
118
|
-
tag: JobTag
|
119
|
-
regex: str
|
120
|
-
|
121
|
-
|
122
|
-
TagFileFilters = typing.Dict[JobTag, TagFileFilter]
|
123
|
-
FilePathSerialise = str
|
124
|
-
FilePathList = typing.List[FilePathSerialise]
|
125
|
-
FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
|
126
|
-
|
127
|
-
|
128
|
-
class JobParamConfig(typing.TypedDict):
|
129
|
-
"""Configures what parameters are passed to the test-callable.
|
130
|
-
|
131
|
-
FIXME: this isn't actually used at all, yet
|
132
|
-
"""
|
133
|
-
|
134
|
-
limitFilesToGroup: bool # whether to limit file-set for the job
|
135
|
-
|
136
|
-
|
137
|
-
class JobAddressConfig(typing.TypedDict):
|
138
|
-
"""Configuration which described a callable to call."""
|
139
|
-
|
140
|
-
file: str # the file-module where 'function' can be found
|
141
|
-
function: str # the 'function' in module to run
|
142
|
-
|
143
|
-
|
144
|
-
class JobContextConfig(typing.TypedDict, total=False):
|
145
|
-
# what parameters the job needs # DEFUNCT
|
146
|
-
params: typing.Optional[JobParamConfig]
|
147
|
-
|
148
|
-
# the path or paths to run the command in. If given a list the job will be
|
149
|
-
# duplicated for each given path.
|
150
|
-
cwd: typing.Optional[typing.Union[str, typing.List[str]]]
|
151
|
-
|
152
|
-
|
153
|
-
class JobWhen(typing.TypedDict, total=False):
|
154
|
-
"""Configures WHEN to call the callable i.e. priority."""
|
155
|
-
|
156
|
-
tags: JobTags # the job tags - used for filtering job-types
|
157
|
-
phase: PhaseName # the phase when the job should be run
|
158
|
-
|
159
|
-
|
160
|
-
class JobWrapper(typing.TypedDict, total=False):
|
161
|
-
"""A base-type for jobs, hooks, and things that can be invoked."""
|
162
|
-
|
163
|
-
addr: JobAddressConfig # which callable to call
|
164
|
-
command: str # a one-liner command to be run
|
165
|
-
|
166
|
-
|
167
|
-
class JobConfig(JobWrapper, total=False):
|
168
|
-
"""A dict that defines a job to be run.
|
169
|
-
|
170
|
-
It consists of the label, address, context and filter information
|
171
|
-
|
172
|
-
TODO: make a class variant of this
|
173
|
-
"""
|
174
|
-
|
175
|
-
label: JobName # the name of the job
|
176
|
-
ctx: typing.Optional[JobContextConfig] # how to call the callable
|
177
|
-
when: JobWhen # when to call the job
|
178
|
-
|
179
|
-
|
180
|
-
Jobs = typing.List[JobConfig]
|
181
|
-
|
182
|
-
PhaseGroupedJobs = typing.DefaultDict[PhaseName, Jobs]
|
183
|
-
|
184
|
-
|
185
|
-
class OptionConfigSerialised(typing.TypedDict):
|
186
|
-
"""Supports better serialisation of options."""
|
187
|
-
|
188
|
-
option: OptionConfig
|
189
|
-
|
190
|
-
|
191
|
-
class TagFileFilterSerialised(typing.TypedDict):
|
192
|
-
"""Supports better serialisation of TagFileFilters."""
|
193
|
-
|
194
|
-
filter: TagFileFilter
|
195
|
-
|
196
|
-
|
197
|
-
class GlobalConfig(typing.TypedDict):
|
198
|
-
"""The config for the entire test run."""
|
199
|
-
|
200
|
-
# Phases control the order of jobs, jobs earlier in the stack get run earlier
|
201
|
-
# the core ide here is to ensure that certain types of job-dependencies,
|
202
|
-
# such as code-reformatting jobs run before analysis tools, therefore making
|
203
|
-
# any error messages about the code give consistent line numbers e..g if a
|
204
|
-
# re-formatter edits a file the error line will move and the analysis phase
|
205
|
-
# will report the wrong line.
|
206
|
-
phases: OrderedPhases
|
207
|
-
|
208
|
-
# Options control the extra flags that are optionally consumed by job.
|
209
|
-
# Options configured here are used to set command-line-options. All options
|
210
|
-
# and their current state are passed to each job.
|
211
|
-
options: typing.Optional[typing.List[OptionConfigSerialised]]
|
212
|
-
|
213
|
-
# File filters control which files will be passed to jobs for a given tags.
|
214
|
-
# Job will receive the super-set of files for all that job's tags.
|
215
|
-
files: typing.Optional[typing.List[TagFileFilterSerialised]]
|
216
|
-
|
217
|
-
# Which minimal version of runem does this config support?
|
218
|
-
min_version: typing.Optional[str]
|
219
|
-
|
220
|
-
|
221
|
-
class GlobalSerialisedConfig(typing.TypedDict):
|
222
|
-
"""Intended to make reading a config file easier.
|
223
|
-
|
224
|
-
Unlike JobSerialisedConfig, this type may not actually help readability.
|
225
|
-
|
226
|
-
An intermediary type for serialisation of the global config, the 'global' resides
|
227
|
-
inside a 'global' key and therefore is easier to find and reason about.
|
228
|
-
"""
|
229
|
-
|
230
|
-
config: GlobalConfig
|
231
|
-
|
232
|
-
|
233
|
-
class HookConfig(JobWrapper, total=False):
|
234
|
-
"""Specification for hooks.
|
235
|
-
|
236
|
-
Like JobConfig with use addr or command to specify what to execute.
|
237
|
-
"""
|
238
|
-
|
239
|
-
hook_name: HookName # the hook for when this is called
|
240
|
-
|
241
|
-
|
242
|
-
Hooks = typing.DefaultDict[HookName, typing.List[HookConfig]]
|
243
|
-
|
244
|
-
# A dictionary to hold hooks, with hook names as keys
|
245
|
-
HooksStore = typing.Dict[HookName, typing.List[HookConfig]]
|
246
|
-
|
247
|
-
|
248
|
-
class HookSerialisedConfig(typing.TypedDict):
|
249
|
-
"""Intended to make reading a config file easier.
|
250
|
-
|
251
|
-
Also, unlike JobSerialisedConfig, this type may not actually help readability.
|
252
|
-
"""
|
253
|
-
|
254
|
-
hook: HookConfig
|
255
|
-
|
256
|
-
|
257
|
-
class JobSerialisedConfig(typing.TypedDict):
|
258
|
-
"""Makes serialised configs easier to read.
|
259
|
-
|
260
|
-
An intermediary typ for serialisation as each 'job' resides inside a 'job' key.
|
261
|
-
|
262
|
-
This makes formatting of YAML config _significantly_ easier to understand.
|
263
|
-
"""
|
264
|
-
|
265
|
-
job: JobConfig
|
266
|
-
|
267
|
-
|
268
|
-
ConfigNodes = typing.Union[
|
269
|
-
GlobalSerialisedConfig, JobSerialisedConfig, HookSerialisedConfig
|
270
|
-
]
|
271
|
-
# The config format as it is serialised to/from disk
|
272
|
-
Config = typing.List[ConfigNodes]
|
273
|
-
|
274
|
-
UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
|
275
|
-
|
276
|
-
|
277
|
-
class CommonKwargs(
|
278
|
-
typing.TypedDict,
|
279
|
-
total=True, # each of these are guaranteed to exist in jobs and hooks
|
280
|
-
):
|
281
|
-
"""Defines the base args that are passed to all jobs.
|
282
|
-
|
283
|
-
As we call hooks and job-task in the same manner, this defines the variables that we
|
284
|
-
can access from both hooks and job-tasks.
|
285
|
-
"""
|
286
|
-
|
287
|
-
root_path: pathlib.Path # the path where the .runem.yml file is
|
288
|
-
job: JobConfig # the job or hook task spec ¢ TODO: rename this
|
289
|
-
label: str # the name of the hook or the job-label
|
290
|
-
options: Options # options passed in on the command line
|
291
|
-
procs: int # the max number of concurrent procs to run
|
292
|
-
verbose: bool # control log verbosity
|
293
|
-
|
294
|
-
|
295
|
-
class HookSpecificKwargs(typing.TypedDict, total=False):
|
296
|
-
"""Defines the args that are passed down to the hooks.
|
297
|
-
|
298
|
-
NOTE: that although these however
|
299
|
-
outside of the *hook* context, the data will not be present. Such is the
|
300
|
-
difficulty in dynamic programming.
|
301
|
-
"""
|
302
|
-
|
303
|
-
wall_clock_time_saved: timedelta # only on `HookName.ON_EXIT`
|
304
|
-
|
305
|
-
|
306
|
-
class JobTaskKwargs(
|
307
|
-
typing.TypedDict,
|
308
|
-
total=False, # for now, we don't enforce these types for job-context, but we should.
|
309
|
-
):
|
310
|
-
"""Defines the task-specific args for job-task functions."""
|
311
|
-
|
312
|
-
file_list: FilePathList
|
313
|
-
record_sub_job_time: typing.Optional[typing.Callable[[str, timedelta], None]]
|
314
|
-
|
315
|
-
|
316
|
-
class HookKwargs(CommonKwargs, HookSpecificKwargs):
|
317
|
-
"""A merged set of kwargs for runem-hooks."""
|
318
|
-
|
319
|
-
pass
|
320
|
-
|
321
|
-
|
322
|
-
class JobKwargs(CommonKwargs, JobTaskKwargs):
|
323
|
-
"""A merged set of kwargs for job-tasks."""
|
324
|
-
|
325
|
-
pass
|
326
|
-
|
327
|
-
|
328
|
-
class AllKwargs(CommonKwargs, JobTaskKwargs, HookSpecificKwargs):
|
329
|
-
"""A merged set of kwargs for al job-functions."""
|
330
|
-
|
331
|
-
pass
|
332
|
-
|
333
|
-
|
334
|
-
@typing.runtime_checkable
|
335
|
-
class JobFunction(typing.Protocol):
|
336
|
-
def __call__(self, **kwargs: Unpack[AllKwargs]) -> JobReturn: # pragma: no cover
|
337
|
-
"""Defines the call() protocol's abstract pattern for job-tasks."""
|
338
|
-
|
339
|
-
@property
|
340
|
-
def __name__(self) -> str: # pragma: no cover
|
341
|
-
"""Defines the name protocol for job-task functions.
|
342
|
-
|
343
|
-
This is primarily used for internal tests but can be useful for introspection.
|
344
|
-
"""
|
345
|
-
|
346
|
-
|
347
|
-
def _hook_example(
|
348
|
-
wall_clock_time_saved: timedelta,
|
349
|
-
**kwargs: typing.Any,
|
350
|
-
) -> None:
|
351
|
-
"""An example hook."""
|
352
|
-
|
353
|
-
|
354
|
-
def _job_task_example(
|
355
|
-
**kwargs: Unpack[JobKwargs],
|
356
|
-
) -> None:
|
357
|
-
"""An example job-task function."""
|
runem-0.1.0.dist-info/RECORD
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
runem/VERSION,sha256=6d2FB_S_DG9CRY5BrqgzrQvT9hJycjNe7pv01YVB7Wc,6
|
2
|
-
runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
-
runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
|
4
|
-
runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
|
5
|
-
runem/blocking_print.py,sha256=S9dtgAeuTzc2-ht-vk9Wl6l-0PwS2tYbHDHDQQitrlA,841
|
6
|
-
runem/cli.py,sha256=wEt_Jnumhl8SiOdKdSJzLkJpWv6n3_Odhi_HeIixr1k,134
|
7
|
-
runem/command_line.py,sha256=mJbs7uUtAxSnbMxO3C08feOkVusK36x_LIQaeJHivzA,13700
|
8
|
-
runem/config.py,sha256=y-e6j84FDiLSKKw9ShDzRlnS5t2e81MW8fKSKtxtJtg,5935
|
9
|
-
runem/config_metadata.py,sha256=Vy7dx8F-Z5jEp16OP2y6vHHoGkyhoCaTG4KIVkMWR7M,3232
|
10
|
-
runem/config_parse.py,sha256=6mCamzWu7HTotmqFJmLZg9FFE6qe1-rpmo8_v5ESPW8,13401
|
11
|
-
runem/files.py,sha256=QZqPS7OA98lEwlhJNtnaSWlEeTlI8_yn-zjf3QAPoJk,4384
|
12
|
-
runem/hook_manager.py,sha256=rhQ4bEWSbn7cfLFtIxJwtaiDe7v1ykvUzmpk9BpZFlM,4241
|
13
|
-
runem/informative_dict.py,sha256=U7p9z78UwOT4TAfng1iDXCEyeYz6C-XZlx9Z1pWNVrI,1548
|
14
|
-
runem/job.py,sha256=QVXvzz67fJk__-h0womFQsB80-w41E3XRcHpxmRnv3o,2912
|
15
|
-
runem/job_execute.py,sha256=M3hV83SKQMl2kOzVghwbUEYa9B2cQNJR7ixOgxNuWKA,4146
|
16
|
-
runem/job_filter.py,sha256=fuxyKCHpTB4HlT_QagBk-IhhmWMlOr9Y9s5voP4yzYU,5370
|
17
|
-
runem/job_runner_simple_command.py,sha256=jxBukPm9bTLNhfQCkqNG5VepvB2ysmWAZwhBPHoTA6o,1091
|
18
|
-
runem/job_wrapper.py,sha256=H57b52Apcg3q3LyMxus6vNrlv615J4CXiWaQrv8vOgY,936
|
19
|
-
runem/job_wrapper_python.py,sha256=m5xbWQxkDRtawjCcgxctzouv_Pny6bKiG2OPVE1hlgo,4226
|
20
|
-
runem/log.py,sha256=dIrocigvIJs1ZGkAzTogXkAK-0ZW3q5FkjpDgLdeW-E,630
|
21
|
-
runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
22
|
-
runem/report.py,sha256=IBCtMgGwnvVbEEqDWbYBGUZmTIzBLnpXqYSK5uu3vgk,8987
|
23
|
-
runem/run_command.py,sha256=Egl_j4bJ9mwi2JEFCsl0W6WH2IRgIdpMN7qdj8voClQ,6386
|
24
|
-
runem/runem.py,sha256=IINSRvp0Yytzh-VNY98hPGmxrjyOndSeH0D2eEuyYZA,14303
|
25
|
-
runem/runem_version.py,sha256=MbETwZO2Tb1Y3hX_OYZjKepEMKA1cjNvr-7Cqhz6e3s,271
|
26
|
-
runem/types.py,sha256=-RHnskpWEtvqNlrHx3mHrkx6ujEdkCxcU7hu3_lHrJU,10671
|
27
|
-
runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
|
28
|
-
runem-0.1.0.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
|
29
|
-
runem-0.1.0.dist-info/METADATA,sha256=PsBSJX4jHev1NegY-LOjmemYGFFAOR8OUY1-nf2mybE,5842
|
30
|
-
runem-0.1.0.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
|
31
|
-
runem-0.1.0.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
|
32
|
-
runem-0.1.0.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
|
33
|
-
runem-0.1.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|