runem 0.0.32__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {runem-0.0.32 → runem-0.1.0}/HISTORY.md +67 -0
- {runem-0.0.32 → runem-0.1.0}/PKG-INFO +8 -7
- {runem-0.0.32 → runem-0.1.0}/requirements-test.txt +5 -5
- runem-0.1.0/requirements.txt +9 -0
- runem-0.1.0/runem/VERSION +1 -0
- {runem-0.0.32 → runem-0.1.0}/runem/command_line.py +42 -1
- {runem-0.0.32 → runem-0.1.0}/runem/hook_manager.py +4 -1
- {runem-0.0.32 → runem-0.1.0}/runem/job_execute.py +10 -6
- {runem-0.0.32 → runem-0.1.0}/runem/job_wrapper.py +1 -1
- {runem-0.0.32 → runem-0.1.0}/runem/runem.py +73 -35
- {runem-0.0.32 → runem-0.1.0}/runem/types.py +112 -7
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/PKG-INFO +8 -7
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/requires.txt +7 -6
- {runem-0.0.32 → runem-0.1.0}/tests/test_runem.py +2 -0
- runem-0.0.32/requirements.txt +0 -3
- runem-0.0.32/runem/VERSION +0 -1
- {runem-0.0.32 → runem-0.1.0}/Containerfile +0 -0
- {runem-0.0.32 → runem-0.1.0}/LICENSE +0 -0
- {runem-0.0.32 → runem-0.1.0}/MANIFEST.in +0 -0
- {runem-0.0.32 → runem-0.1.0}/README.md +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/__init__.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/__main__.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/base.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/blocking_print.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/cli/initialise_options.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/cli.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/config.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/config_metadata.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/config_parse.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/files.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/informative_dict.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/job.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/job_filter.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/job_runner_simple_command.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/job_wrapper_python.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/log.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/py.typed +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/report.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/run_command.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/runem_version.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem/utils.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/SOURCES.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/dependency_links.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/entry_points.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/runem.egg-info/top_level.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/setup.cfg +0 -0
- {runem-0.0.32 → runem-0.1.0}/setup.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/__init__.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/cli/test_initialise_options.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/conftest.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/data/help_output.3.10.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/data/help_output.3.11.txt +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/intentional_test_error.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/sanitise_reports_footer.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_base.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_blocking_print.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_cli.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_config.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_config_parse.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_files.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_hook_manager.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_informative_dict.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job_execute.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job_filter.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job_runner_simple_command.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job_wrapper.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_job_wrapper_python.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_report.py +0 -0
- {runem-0.0.32 → runem-0.1.0}/tests/test_run_command.py +0 -0
@@ -4,6 +4,73 @@ Changelog
|
|
4
4
|
|
5
5
|
(unreleased)
|
6
6
|
------------
|
7
|
+
- Merge pull request #57 from lursight/feat/replace_halo_with_rich.
|
8
|
+
[Frank Harrison]
|
9
|
+
|
10
|
+
feat(rich): moves to using rich instead of halo for spinners
|
11
|
+
- Feat(rich): moves to using rich instead of halo for spinners. [Frank
|
12
|
+
Harrison]
|
13
|
+
- Merge pull request #56 from lursight/feat/better_job_function_typing.
|
14
|
+
[Frank Harrison]
|
15
|
+
|
16
|
+
feat(better-job-function-typing): adds stronger typing for job and hook tasks
|
17
|
+
- Feat(better-job-function-typing): adds stronger typing for job and
|
18
|
+
hook tasks. [Frank Harrison]
|
19
|
+
|
20
|
+
We do this by typing the kwargs convenience variable, for which we need
|
21
|
+
to use `Unpack` (for back compatibility), and `type_extensions` for
|
22
|
+
cross-python-version compatibility (i.e. something that work for all
|
23
|
+
targeted version of python).
|
24
|
+
|
25
|
+
As a side-effect of this we get the benefit of seeing when and where we
|
26
|
+
add extra data into the call-stack for `job_execute()` and it emerges
|
27
|
+
that we only extend the key-word args when we are calling hooks in a
|
28
|
+
non-threaded way, which makes sense.
|
29
|
+
|
30
|
+
We do several things to achieve this:
|
31
|
+
1. We have common parameters passed to both hooks and job-tasks
|
32
|
+
- These common parameters make the hooks and job-task feel similar
|
33
|
+
to develop.
|
34
|
+
2. We put all hook-specific kwargs in one place, and mark each as
|
35
|
+
optional.
|
36
|
+
- this, for now, is mainly because we only have one hook, so this
|
37
|
+
will likely change.
|
38
|
+
3. We share and combine kwargs in a range of inheritance types, mainly
|
39
|
+
to work with(/around?) python-typing, which isn't great in this
|
40
|
+
type of situation.
|
41
|
+
- Chore(ignores): adds the tox/ dir to the ignores. [Frank Harrison]
|
42
|
+
- Chore(ignores): update git ignore for coverage files and docs gen.
|
43
|
+
[Frank Harrison]
|
44
|
+
|
45
|
+
The docs are recent additions.
|
46
|
+
- Merge pull request #55 from lursight/chore/test_improvements. [Frank
|
47
|
+
Harrison]
|
48
|
+
|
49
|
+
Chore/test improvements
|
50
|
+
- Chore(coverage): reduce false-positives by deleteing old
|
51
|
+
coverage_report files. [Frank Harrison]
|
52
|
+
|
53
|
+
Sometimes we would have stale .coverage_report.* files left behind where
|
54
|
+
from the multi-distributed pytest runs. These would lead more lines
|
55
|
+
being reported as coverage than actual - aka false-positive test passes
|
56
|
+
for coverage.
|
57
|
+
- Chore(help-tests): fixes the help output width in tests. [Frank
|
58
|
+
Harrison]
|
59
|
+
|
60
|
+
This reduces false-negative test results where we get word-splits in
|
61
|
+
directories or between process counters or other dynamic content.
|
62
|
+
|
63
|
+
This was due to the width of the terminal when the test was being run
|
64
|
+
being variant with developer-machine. This uses a fixed-width output,
|
65
|
+
reducing, if not stopping issues.
|
66
|
+
- Chore(deps): updates black 24.3.0 -> 24.10.0. [Frank Harrison]
|
67
|
+
- Chore(deps): updates to latest pytest 8.1.1 -> 8.3.3 + plugins. [Frank
|
68
|
+
Harrison]
|
69
|
+
|
70
|
+
|
71
|
+
0.0.32 (2024-11-17)
|
72
|
+
-------------------
|
73
|
+
- Release: version 0.0.32 🚀 [Frank Harrison]
|
7
74
|
- Merge pull request #54 from lursight/chore/use_tox_on_release. [Frank
|
8
75
|
Harrison]
|
9
76
|
|
@@ -1,17 +1,18 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: runem
|
3
|
-
Version: 0.0
|
3
|
+
Version: 0.1.0
|
4
4
|
Summary: Awesome runem created by lursight
|
5
5
|
Home-page: https://github.com/lursight/runem/
|
6
6
|
Author: lursight
|
7
7
|
Description-Content-Type: text/markdown
|
8
8
|
License-File: LICENSE
|
9
|
-
Requires-Dist: halo
|
10
9
|
Requires-Dist: packaging
|
11
10
|
Requires-Dist: PyYAML
|
11
|
+
Requires-Dist: rich
|
12
|
+
Requires-Dist: typing_extensions
|
12
13
|
Provides-Extra: tests
|
13
|
-
Requires-Dist: black==24.
|
14
|
-
Requires-Dist: coverage==7.
|
14
|
+
Requires-Dist: black==24.10.0; extra == "tests"
|
15
|
+
Requires-Dist: coverage==7.5; extra == "tests"
|
15
16
|
Requires-Dist: docformatter==1.7.5; extra == "tests"
|
16
17
|
Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
|
17
18
|
Requires-Dist: flake8==7.0.0; extra == "tests"
|
@@ -22,10 +23,10 @@ Requires-Dist: mypy==1.9.0; extra == "tests"
|
|
22
23
|
Requires-Dist: pydocstyle==6.3.0; extra == "tests"
|
23
24
|
Requires-Dist: pylint==3.1.0; extra == "tests"
|
24
25
|
Requires-Dist: pylama==8.4.1; extra == "tests"
|
25
|
-
Requires-Dist: pytest-cov==
|
26
|
+
Requires-Dist: pytest-cov==6.0.0; extra == "tests"
|
26
27
|
Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
|
27
|
-
Requires-Dist: pytest-xdist==3.
|
28
|
-
Requires-Dist: pytest==8.
|
28
|
+
Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
|
29
|
+
Requires-Dist: pytest==8.3.3; extra == "tests"
|
29
30
|
Requires-Dist: setuptools; extra == "tests"
|
30
31
|
Requires-Dist: termplotlib==0.3.9; extra == "tests"
|
31
32
|
Requires-Dist: tox; extra == "tests"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# This requirements are for development and testing only, not for production.
|
2
|
-
black==24.
|
3
|
-
coverage==7.
|
2
|
+
black==24.10.0
|
3
|
+
coverage==7.5
|
4
4
|
docformatter==1.7.5
|
5
5
|
flake8-bugbear==24.2.6
|
6
6
|
flake8==7.0.0
|
@@ -11,10 +11,10 @@ mypy==1.9.0
|
|
11
11
|
pydocstyle==6.3.0
|
12
12
|
pylint==3.1.0
|
13
13
|
pylama==8.4.1
|
14
|
-
pytest-cov==
|
14
|
+
pytest-cov==6.0.0
|
15
15
|
pytest-profiling==1.7.0
|
16
|
-
pytest-xdist==3.
|
17
|
-
pytest==8.
|
16
|
+
pytest-xdist==3.6.1
|
17
|
+
pytest==8.3.3
|
18
18
|
setuptools
|
19
19
|
termplotlib==0.3.9
|
20
20
|
tox
|
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -12,6 +12,45 @@ from runem.types import JobNames, OptionConfig, OptionsWritable
|
|
12
12
|
from runem.utils import printable_set
|
13
13
|
|
14
14
|
|
15
|
+
class HelpFormatterFixedWidth(argparse.HelpFormatter):
|
16
|
+
"""This works around test issues via constant width helo output.
|
17
|
+
|
18
|
+
This ensures that we get more constant for help-text by fixing the width to
|
19
|
+
something reasonable.
|
20
|
+
"""
|
21
|
+
|
22
|
+
def __init__(self, prog: typing.Any) -> None:
|
23
|
+
# Override the default width with a fixed width, for tests.
|
24
|
+
super().__init__(
|
25
|
+
prog,
|
26
|
+
# Pretty wide so we do not get wrapping on directories
|
27
|
+
# or process-count output.
|
28
|
+
width=1000,
|
29
|
+
)
|
30
|
+
|
31
|
+
|
32
|
+
def _get_argparse_help_formatter() -> typing.Any:
|
33
|
+
"""Returns a help-formatter for argparse.
|
34
|
+
|
35
|
+
This is for tests only to fake terminals of a constant with when rendering help
|
36
|
+
output.
|
37
|
+
"""
|
38
|
+
# Check environment variable to see if we're in tests and need a fixed width
|
39
|
+
# help output.
|
40
|
+
use_fixed_width = os.getenv("RUNEM_FIXED_HELP_WIDTH", None)
|
41
|
+
|
42
|
+
if use_fixed_width:
|
43
|
+
# Use custom formatter with the width specified in the environment variable
|
44
|
+
return (
|
45
|
+
lambda prog: HelpFormatterFixedWidth( # pylint: disable=unnecessary-lambda
|
46
|
+
prog
|
47
|
+
)
|
48
|
+
)
|
49
|
+
|
50
|
+
# Use default formatter
|
51
|
+
return argparse.HelpFormatter
|
52
|
+
|
53
|
+
|
15
54
|
def parse_args(
|
16
55
|
config_metadata: ConfigMetadata, argv: typing.List[str]
|
17
56
|
) -> ConfigMetadata:
|
@@ -23,7 +62,9 @@ def parse_args(
|
|
23
62
|
Returns the parsed args, the jobs_names_to_run, job_phases_to_run, job_tags_to_run
|
24
63
|
"""
|
25
64
|
parser = argparse.ArgumentParser(
|
26
|
-
add_help=False,
|
65
|
+
add_help=False,
|
66
|
+
description="Runs the Lursight Lang test-suite",
|
67
|
+
formatter_class=_get_argparse_help_formatter(),
|
27
68
|
)
|
28
69
|
parser.add_argument(
|
29
70
|
"-H", "--help", action="help", help="show this help message and exit"
|
@@ -1,6 +1,8 @@
|
|
1
1
|
import typing
|
2
2
|
from collections import defaultdict
|
3
3
|
|
4
|
+
from typing_extensions import Unpack
|
5
|
+
|
4
6
|
from runem.config_metadata import ConfigMetadata
|
5
7
|
from runem.job import Job
|
6
8
|
from runem.job_execute import job_execute
|
@@ -10,6 +12,7 @@ from runem.types import (
|
|
10
12
|
HookConfig,
|
11
13
|
HookName,
|
12
14
|
Hooks,
|
15
|
+
HookSpecificKwargs,
|
13
16
|
HooksStore,
|
14
17
|
JobConfig,
|
15
18
|
)
|
@@ -69,7 +72,7 @@ class HookManager:
|
|
69
72
|
self,
|
70
73
|
hook_name: HookName,
|
71
74
|
config_metadata: ConfigMetadata,
|
72
|
-
**kwargs:
|
75
|
+
**kwargs: Unpack[HookSpecificKwargs],
|
73
76
|
) -> None:
|
74
77
|
"""Invokes all functions registered to a specific hook."""
|
75
78
|
hooks: typing.List[HookConfig] = self.hooks_store.get(hook_name, [])
|
@@ -5,13 +5,17 @@ import uuid
|
|
5
5
|
from datetime import timedelta
|
6
6
|
from timeit import default_timer as timer
|
7
7
|
|
8
|
+
from typing_extensions import Unpack
|
9
|
+
|
8
10
|
from runem.config_metadata import ConfigMetadata
|
9
11
|
from runem.informative_dict import ReadOnlyInformativeDict
|
10
12
|
from runem.job import Job
|
11
13
|
from runem.job_wrapper import get_job_wrapper
|
12
14
|
from runem.log import error, log
|
13
15
|
from runem.types import (
|
16
|
+
FilePathList,
|
14
17
|
FilePathListLookup,
|
18
|
+
HookSpecificKwargs,
|
15
19
|
JobConfig,
|
16
20
|
JobFunction,
|
17
21
|
JobReturn,
|
@@ -26,7 +30,7 @@ def job_execute_inner(
|
|
26
30
|
job_config: JobConfig,
|
27
31
|
config_metadata: ConfigMetadata,
|
28
32
|
file_lists: FilePathListLookup,
|
29
|
-
**kwargs:
|
33
|
+
**kwargs: Unpack[HookSpecificKwargs],
|
30
34
|
) -> typing.Tuple[JobTiming, JobReturn]:
|
31
35
|
"""Wrapper for running a job inside a sub-process.
|
32
36
|
|
@@ -36,13 +40,12 @@ def job_execute_inner(
|
|
36
40
|
if config_metadata.args.verbose:
|
37
41
|
log(f"START: '{label}'")
|
38
42
|
root_path: pathlib.Path = config_metadata.cfg_filepath.parent
|
39
|
-
function: JobFunction
|
40
43
|
job_tags: typing.Optional[JobTags] = Job.get_job_tags(job_config)
|
41
44
|
os.chdir(root_path)
|
42
|
-
function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
|
45
|
+
function: JobFunction = get_job_wrapper(job_config, config_metadata.cfg_filepath)
|
43
46
|
|
44
47
|
# get the files for all files found for this job's tags
|
45
|
-
file_list = Job.get_job_files(file_lists, job_tags)
|
48
|
+
file_list: FilePathList = Job.get_job_files(file_lists, job_tags)
|
46
49
|
|
47
50
|
if not file_list:
|
48
51
|
# no files to work on
|
@@ -77,8 +80,9 @@ def job_execute_inner(
|
|
77
80
|
log(f"job: running: '{Job.get_job_name(job_config)}'")
|
78
81
|
reports: JobReturn
|
79
82
|
try:
|
83
|
+
assert isinstance(function, JobFunction)
|
80
84
|
reports = function(
|
81
|
-
options=ReadOnlyInformativeDict(config_metadata.options),
|
85
|
+
options=ReadOnlyInformativeDict(config_metadata.options),
|
82
86
|
file_list=file_list,
|
83
87
|
procs=config_metadata.args.procs,
|
84
88
|
root_path=root_path,
|
@@ -109,7 +113,7 @@ def job_execute(
|
|
109
113
|
running_jobs: typing.Dict[str, str],
|
110
114
|
config_metadata: ConfigMetadata,
|
111
115
|
file_lists: FilePathListLookup,
|
112
|
-
**kwargs:
|
116
|
+
**kwargs: Unpack[HookSpecificKwargs],
|
113
117
|
) -> typing.Tuple[JobTiming, JobReturn]:
|
114
118
|
"""Thin-wrapper around job_execute_inner needed for mocking in tests.
|
115
119
|
|
@@ -18,7 +18,7 @@ def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobF
|
|
18
18
|
# validate that the command is "understandable" and usable.
|
19
19
|
command_string: str = job_wrapper["command"]
|
20
20
|
validate_simple_command(command_string)
|
21
|
-
return job_runner_simple_command
|
21
|
+
return job_runner_simple_command
|
22
22
|
|
23
23
|
# if we do not have a simple command address assume we have just an addressed
|
24
24
|
# function
|
@@ -30,8 +30,11 @@ from datetime import timedelta
|
|
30
30
|
from itertools import repeat
|
31
31
|
from multiprocessing.managers import DictProxy, ListProxy, ValueProxy
|
32
32
|
from timeit import default_timer as timer
|
33
|
+
from types import TracebackType
|
33
34
|
|
34
|
-
from
|
35
|
+
from rich.console import Console, ConsoleOptions, ConsoleRenderable, RenderResult
|
36
|
+
from rich.spinner import Spinner
|
37
|
+
from rich.text import Text
|
35
38
|
|
36
39
|
from runem.command_line import parse_args
|
37
40
|
from runem.config import load_project_config, load_user_configs
|
@@ -58,6 +61,8 @@ from runem.types import (
|
|
58
61
|
)
|
59
62
|
from runem.utils import printable_set
|
60
63
|
|
64
|
+
rich_console = Console()
|
65
|
+
|
61
66
|
|
62
67
|
def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
|
63
68
|
"""Loads config, parsing cli input and produces the run config.
|
@@ -85,6 +90,36 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
|
|
85
90
|
return config_metadata
|
86
91
|
|
87
92
|
|
93
|
+
class DummySpinner(ConsoleRenderable): # pragma: no cover
|
94
|
+
"""A dummy spinner for when spinners are disabled."""
|
95
|
+
|
96
|
+
def __init__(self) -> None:
|
97
|
+
self.text = ""
|
98
|
+
|
99
|
+
def __rich__(self) -> Text:
|
100
|
+
"""Return a rich Text object for rendering."""
|
101
|
+
return Text(self.text)
|
102
|
+
|
103
|
+
def __rich_console__(
|
104
|
+
self, console: Console, options: ConsoleOptions
|
105
|
+
) -> RenderResult:
|
106
|
+
"""Yield an empty string or placeholder text."""
|
107
|
+
yield Text(self.text)
|
108
|
+
|
109
|
+
def __enter__(self) -> None:
|
110
|
+
"""Support for context manager."""
|
111
|
+
pass
|
112
|
+
|
113
|
+
def __exit__(
|
114
|
+
self,
|
115
|
+
exc_type: typing.Optional[typing.Type[BaseException]],
|
116
|
+
exc_value: typing.Optional[BaseException],
|
117
|
+
traceback: typing.Optional[TracebackType],
|
118
|
+
) -> None:
|
119
|
+
"""Support for context manager."""
|
120
|
+
pass
|
121
|
+
|
122
|
+
|
88
123
|
def _update_progress(
|
89
124
|
label: str,
|
90
125
|
running_jobs: typing.Dict[str, str],
|
@@ -104,52 +139,55 @@ def _update_progress(
|
|
104
139
|
is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
|
105
140
|
num_workers (int): Indicates the number of workers performing the jobs.
|
106
141
|
"""
|
107
|
-
# Using
|
142
|
+
# Using the `rich` module to show a loading spinner on console
|
143
|
+
spinner: typing.Union[Spinner, DummySpinner]
|
108
144
|
if show_spinner:
|
109
|
-
spinner =
|
110
|
-
|
145
|
+
spinner = Spinner("dots", text="Starting tasks...")
|
146
|
+
else:
|
147
|
+
spinner = DummySpinner()
|
111
148
|
|
112
|
-
|
113
|
-
all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
|
114
|
-
completed_jobs: typing.Set[str] = set()
|
149
|
+
with rich_console.status(spinner):
|
115
150
|
|
116
|
-
|
117
|
-
|
151
|
+
# The set of all job labels, and the set of completed jobs
|
152
|
+
all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
|
153
|
+
completed_jobs: typing.Set[str] = set()
|
118
154
|
|
119
|
-
|
120
|
-
|
121
|
-
seen_jobs = list(running_jobs_set.union(seen_jobs)) # Update the seen jobs
|
155
|
+
# This dataset is used to track changes between iterations
|
156
|
+
last_running_jobs_set: typing.Set[str] = set()
|
122
157
|
|
123
|
-
|
124
|
-
|
158
|
+
while is_running.value:
|
159
|
+
running_jobs_set: typing.Set[str] = set(running_jobs.values())
|
160
|
+
seen_jobs = list(running_jobs_set.union(seen_jobs)) # Update the seen jobs
|
125
161
|
|
126
|
-
|
127
|
-
|
162
|
+
# Jobs that have disappeared since last check
|
163
|
+
disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
|
128
164
|
|
129
|
-
|
130
|
-
|
165
|
+
# Jobs that have not yet completed
|
166
|
+
remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
|
131
167
|
|
132
|
-
|
133
|
-
|
134
|
-
all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
|
135
|
-
disappeared_jobs.update(all_completed_jobs - running_jobs_set)
|
168
|
+
# Check if we're closing to completion
|
169
|
+
workers_retiring: bool = len(remaining_jobs) <= num_workers
|
136
170
|
|
137
|
-
|
171
|
+
if workers_retiring:
|
172
|
+
# Handle edge case: a task may have disappeared whilst process was sleeping
|
173
|
+
all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
|
174
|
+
disappeared_jobs.update(all_completed_jobs - running_jobs_set)
|
138
175
|
|
139
|
-
|
140
|
-
progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
|
141
|
-
running_jobs_list = printable_set(running_jobs_set)
|
142
|
-
if show_spinner:
|
143
|
-
spinner.text = f"{label}: {progress}({num_workers}): {running_jobs_list}"
|
176
|
+
completed_jobs.update(disappeared_jobs)
|
144
177
|
|
145
|
-
|
146
|
-
|
178
|
+
# Prepare progress report
|
179
|
+
progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
|
180
|
+
running_jobs_list = printable_set(running_jobs_set)
|
181
|
+
if show_spinner:
|
182
|
+
spinner.text = (
|
183
|
+
f"{label}: {progress}({num_workers}): {running_jobs_list}"
|
184
|
+
)
|
147
185
|
|
148
|
-
|
149
|
-
|
186
|
+
# Update the tracked dataset for the next iteration
|
187
|
+
last_running_jobs_set = running_jobs_set
|
150
188
|
|
151
|
-
|
152
|
-
|
189
|
+
# Sleep to decrease frequency of updates and reduce CPU usage
|
190
|
+
time.sleep(0.1)
|
153
191
|
|
154
192
|
|
155
193
|
def _process_jobs(
|
@@ -209,7 +247,7 @@ def _process_jobs(
|
|
209
247
|
with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
|
210
248
|
# use starmap so we can pass down the job-configs and the args and the files
|
211
249
|
in_out_job_run_metadatas[phase] = pool.starmap(
|
212
|
-
job_execute,
|
250
|
+
job_execute, # no kwargs passed for jobs here
|
213
251
|
zip(
|
214
252
|
jobs,
|
215
253
|
repeat(running_jobs),
|
@@ -1,9 +1,37 @@
|
|
1
|
-
|
1
|
+
"""
|
2
|
+
|
3
|
+
Some note on Unpack and kwargs:
|
4
|
+
We *try* to strongly type `**kwargs` for clarity.
|
5
|
+
We have tried several ways to define a Generic type that encapsulates
|
6
|
+
`**kwargs: SingleType`
|
7
|
+
... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
|
8
|
+
so we have to recommend instead using:
|
9
|
+
`**kwargs: Unpack[KwArgsType]`
|
10
|
+
|
11
|
+
For this to work across versions of python where support for Unpack changes;
|
12
|
+
for example `Unpack` is a python 3.12 feature, but available in the
|
13
|
+
`typing_extensions` module.
|
14
|
+
|
15
|
+
So, for now, it looks like we get away with importing `Unpack` from the
|
16
|
+
`typing_extensions` module, even in python 3.12, so we will use, and
|
17
|
+
recommend using, the `typing_extensions` of `Unpack`, until it becomes
|
18
|
+
obsolete.
|
19
|
+
|
20
|
+
Alternatively, we can use the following, but it's unnecessarily verbose.
|
21
|
+
|
22
|
+
if sys.version_info >= (3, 12): # pragma: no coverage
|
23
|
+
from typing import Unpack
|
24
|
+
else: # pragma: no coverage
|
25
|
+
from typing_extensions import Unpack
|
26
|
+
"""
|
27
|
+
|
2
28
|
import pathlib
|
3
29
|
import typing
|
4
30
|
from datetime import timedelta
|
5
31
|
from enum import Enum
|
6
32
|
|
33
|
+
from typing_extensions import Unpack
|
34
|
+
|
7
35
|
from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
|
8
36
|
|
9
37
|
|
@@ -96,12 +124,6 @@ FilePathSerialise = str
|
|
96
124
|
FilePathList = typing.List[FilePathSerialise]
|
97
125
|
FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
|
98
126
|
|
99
|
-
# FIXME: this type is no-longer the actual spec of the test-functions
|
100
|
-
JobFunction = typing.Union[
|
101
|
-
typing.Callable[[argparse.Namespace, OptionsWritable, FilePathList], None],
|
102
|
-
typing.Callable[[typing.Any], None],
|
103
|
-
]
|
104
|
-
|
105
127
|
|
106
128
|
class JobParamConfig(typing.TypedDict):
|
107
129
|
"""Configures what parameters are passed to the test-callable.
|
@@ -250,3 +272,86 @@ ConfigNodes = typing.Union[
|
|
250
272
|
Config = typing.List[ConfigNodes]
|
251
273
|
|
252
274
|
UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
|
275
|
+
|
276
|
+
|
277
|
+
class CommonKwargs(
|
278
|
+
typing.TypedDict,
|
279
|
+
total=True, # each of these are guaranteed to exist in jobs and hooks
|
280
|
+
):
|
281
|
+
"""Defines the base args that are passed to all jobs.
|
282
|
+
|
283
|
+
As we call hooks and job-task in the same manner, this defines the variables that we
|
284
|
+
can access from both hooks and job-tasks.
|
285
|
+
"""
|
286
|
+
|
287
|
+
root_path: pathlib.Path # the path where the .runem.yml file is
|
288
|
+
job: JobConfig # the job or hook task spec ¢ TODO: rename this
|
289
|
+
label: str # the name of the hook or the job-label
|
290
|
+
options: Options # options passed in on the command line
|
291
|
+
procs: int # the max number of concurrent procs to run
|
292
|
+
verbose: bool # control log verbosity
|
293
|
+
|
294
|
+
|
295
|
+
class HookSpecificKwargs(typing.TypedDict, total=False):
|
296
|
+
"""Defines the args that are passed down to the hooks.
|
297
|
+
|
298
|
+
NOTE: that although these however
|
299
|
+
outside of the *hook* context, the data will not be present. Such is the
|
300
|
+
difficulty in dynamic programming.
|
301
|
+
"""
|
302
|
+
|
303
|
+
wall_clock_time_saved: timedelta # only on `HookName.ON_EXIT`
|
304
|
+
|
305
|
+
|
306
|
+
class JobTaskKwargs(
|
307
|
+
typing.TypedDict,
|
308
|
+
total=False, # for now, we don't enforce these types for job-context, but we should.
|
309
|
+
):
|
310
|
+
"""Defines the task-specific args for job-task functions."""
|
311
|
+
|
312
|
+
file_list: FilePathList
|
313
|
+
record_sub_job_time: typing.Optional[typing.Callable[[str, timedelta], None]]
|
314
|
+
|
315
|
+
|
316
|
+
class HookKwargs(CommonKwargs, HookSpecificKwargs):
|
317
|
+
"""A merged set of kwargs for runem-hooks."""
|
318
|
+
|
319
|
+
pass
|
320
|
+
|
321
|
+
|
322
|
+
class JobKwargs(CommonKwargs, JobTaskKwargs):
|
323
|
+
"""A merged set of kwargs for job-tasks."""
|
324
|
+
|
325
|
+
pass
|
326
|
+
|
327
|
+
|
328
|
+
class AllKwargs(CommonKwargs, JobTaskKwargs, HookSpecificKwargs):
|
329
|
+
"""A merged set of kwargs for al job-functions."""
|
330
|
+
|
331
|
+
pass
|
332
|
+
|
333
|
+
|
334
|
+
@typing.runtime_checkable
|
335
|
+
class JobFunction(typing.Protocol):
|
336
|
+
def __call__(self, **kwargs: Unpack[AllKwargs]) -> JobReturn: # pragma: no cover
|
337
|
+
"""Defines the call() protocol's abstract pattern for job-tasks."""
|
338
|
+
|
339
|
+
@property
|
340
|
+
def __name__(self) -> str: # pragma: no cover
|
341
|
+
"""Defines the name protocol for job-task functions.
|
342
|
+
|
343
|
+
This is primarily used for internal tests but can be useful for introspection.
|
344
|
+
"""
|
345
|
+
|
346
|
+
|
347
|
+
def _hook_example(
|
348
|
+
wall_clock_time_saved: timedelta,
|
349
|
+
**kwargs: typing.Any,
|
350
|
+
) -> None:
|
351
|
+
"""An example hook."""
|
352
|
+
|
353
|
+
|
354
|
+
def _job_task_example(
|
355
|
+
**kwargs: Unpack[JobKwargs],
|
356
|
+
) -> None:
|
357
|
+
"""An example job-task function."""
|
@@ -1,17 +1,18 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: runem
|
3
|
-
Version: 0.0
|
3
|
+
Version: 0.1.0
|
4
4
|
Summary: Awesome runem created by lursight
|
5
5
|
Home-page: https://github.com/lursight/runem/
|
6
6
|
Author: lursight
|
7
7
|
Description-Content-Type: text/markdown
|
8
8
|
License-File: LICENSE
|
9
|
-
Requires-Dist: halo
|
10
9
|
Requires-Dist: packaging
|
11
10
|
Requires-Dist: PyYAML
|
11
|
+
Requires-Dist: rich
|
12
|
+
Requires-Dist: typing_extensions
|
12
13
|
Provides-Extra: tests
|
13
|
-
Requires-Dist: black==24.
|
14
|
-
Requires-Dist: coverage==7.
|
14
|
+
Requires-Dist: black==24.10.0; extra == "tests"
|
15
|
+
Requires-Dist: coverage==7.5; extra == "tests"
|
15
16
|
Requires-Dist: docformatter==1.7.5; extra == "tests"
|
16
17
|
Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
|
17
18
|
Requires-Dist: flake8==7.0.0; extra == "tests"
|
@@ -22,10 +23,10 @@ Requires-Dist: mypy==1.9.0; extra == "tests"
|
|
22
23
|
Requires-Dist: pydocstyle==6.3.0; extra == "tests"
|
23
24
|
Requires-Dist: pylint==3.1.0; extra == "tests"
|
24
25
|
Requires-Dist: pylama==8.4.1; extra == "tests"
|
25
|
-
Requires-Dist: pytest-cov==
|
26
|
+
Requires-Dist: pytest-cov==6.0.0; extra == "tests"
|
26
27
|
Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
|
27
|
-
Requires-Dist: pytest-xdist==3.
|
28
|
-
Requires-Dist: pytest==8.
|
28
|
+
Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
|
29
|
+
Requires-Dist: pytest==8.3.3; extra == "tests"
|
29
30
|
Requires-Dist: setuptools; extra == "tests"
|
30
31
|
Requires-Dist: termplotlib==0.3.9; extra == "tests"
|
31
32
|
Requires-Dist: tox; extra == "tests"
|
@@ -1,10 +1,11 @@
|
|
1
|
-
halo
|
2
1
|
packaging
|
3
2
|
PyYAML
|
3
|
+
rich
|
4
|
+
typing_extensions
|
4
5
|
|
5
6
|
[tests]
|
6
|
-
black==24.
|
7
|
-
coverage==7.
|
7
|
+
black==24.10.0
|
8
|
+
coverage==7.5
|
8
9
|
docformatter==1.7.5
|
9
10
|
flake8-bugbear==24.2.6
|
10
11
|
flake8==7.0.0
|
@@ -15,10 +16,10 @@ mypy==1.9.0
|
|
15
16
|
pydocstyle==6.3.0
|
16
17
|
pylint==3.1.0
|
17
18
|
pylama==8.4.1
|
18
|
-
pytest-cov==
|
19
|
+
pytest-cov==6.0.0
|
19
20
|
pytest-profiling==1.7.0
|
20
|
-
pytest-xdist==3.
|
21
|
-
pytest==8.
|
21
|
+
pytest-xdist==3.6.1
|
22
|
+
pytest==8.3.3
|
22
23
|
setuptools
|
23
24
|
termplotlib==0.3.9
|
24
25
|
tox
|
@@ -669,6 +669,8 @@ def test_runem_help(
|
|
669
669
|
As we build features we want to ensure that the help output stays consistent as we
|
670
670
|
leverage the argparse system to generate the help for a specific .runem.yml config
|
671
671
|
"""
|
672
|
+
# Ensure we get fixed-width output.
|
673
|
+
os.environ["RUNEM_FIXED_HELP_WIDTH"] = "1"
|
672
674
|
runem_cli_switches: typing.List[str] = ["--help"]
|
673
675
|
runem_stdout: typing.List[str]
|
674
676
|
error_raised: typing.Optional[BaseException]
|
runem-0.0.32/requirements.txt
DELETED
runem-0.0.32/runem/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.0.32
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|