runem 0.0.32__tar.gz → 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. {runem-0.0.32 → runem-0.1.0}/HISTORY.md +67 -0
  2. {runem-0.0.32 → runem-0.1.0}/PKG-INFO +8 -7
  3. {runem-0.0.32 → runem-0.1.0}/requirements-test.txt +5 -5
  4. runem-0.1.0/requirements.txt +9 -0
  5. runem-0.1.0/runem/VERSION +1 -0
  6. {runem-0.0.32 → runem-0.1.0}/runem/command_line.py +42 -1
  7. {runem-0.0.32 → runem-0.1.0}/runem/hook_manager.py +4 -1
  8. {runem-0.0.32 → runem-0.1.0}/runem/job_execute.py +10 -6
  9. {runem-0.0.32 → runem-0.1.0}/runem/job_wrapper.py +1 -1
  10. {runem-0.0.32 → runem-0.1.0}/runem/runem.py +73 -35
  11. {runem-0.0.32 → runem-0.1.0}/runem/types.py +112 -7
  12. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/PKG-INFO +8 -7
  13. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/requires.txt +7 -6
  14. {runem-0.0.32 → runem-0.1.0}/tests/test_runem.py +2 -0
  15. runem-0.0.32/requirements.txt +0 -3
  16. runem-0.0.32/runem/VERSION +0 -1
  17. {runem-0.0.32 → runem-0.1.0}/Containerfile +0 -0
  18. {runem-0.0.32 → runem-0.1.0}/LICENSE +0 -0
  19. {runem-0.0.32 → runem-0.1.0}/MANIFEST.in +0 -0
  20. {runem-0.0.32 → runem-0.1.0}/README.md +0 -0
  21. {runem-0.0.32 → runem-0.1.0}/runem/__init__.py +0 -0
  22. {runem-0.0.32 → runem-0.1.0}/runem/__main__.py +0 -0
  23. {runem-0.0.32 → runem-0.1.0}/runem/base.py +0 -0
  24. {runem-0.0.32 → runem-0.1.0}/runem/blocking_print.py +0 -0
  25. {runem-0.0.32 → runem-0.1.0}/runem/cli/initialise_options.py +0 -0
  26. {runem-0.0.32 → runem-0.1.0}/runem/cli.py +0 -0
  27. {runem-0.0.32 → runem-0.1.0}/runem/config.py +0 -0
  28. {runem-0.0.32 → runem-0.1.0}/runem/config_metadata.py +0 -0
  29. {runem-0.0.32 → runem-0.1.0}/runem/config_parse.py +0 -0
  30. {runem-0.0.32 → runem-0.1.0}/runem/files.py +0 -0
  31. {runem-0.0.32 → runem-0.1.0}/runem/informative_dict.py +0 -0
  32. {runem-0.0.32 → runem-0.1.0}/runem/job.py +0 -0
  33. {runem-0.0.32 → runem-0.1.0}/runem/job_filter.py +0 -0
  34. {runem-0.0.32 → runem-0.1.0}/runem/job_runner_simple_command.py +0 -0
  35. {runem-0.0.32 → runem-0.1.0}/runem/job_wrapper_python.py +0 -0
  36. {runem-0.0.32 → runem-0.1.0}/runem/log.py +0 -0
  37. {runem-0.0.32 → runem-0.1.0}/runem/py.typed +0 -0
  38. {runem-0.0.32 → runem-0.1.0}/runem/report.py +0 -0
  39. {runem-0.0.32 → runem-0.1.0}/runem/run_command.py +0 -0
  40. {runem-0.0.32 → runem-0.1.0}/runem/runem_version.py +0 -0
  41. {runem-0.0.32 → runem-0.1.0}/runem/utils.py +0 -0
  42. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/SOURCES.txt +0 -0
  43. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/dependency_links.txt +0 -0
  44. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/entry_points.txt +0 -0
  45. {runem-0.0.32 → runem-0.1.0}/runem.egg-info/top_level.txt +0 -0
  46. {runem-0.0.32 → runem-0.1.0}/setup.cfg +0 -0
  47. {runem-0.0.32 → runem-0.1.0}/setup.py +0 -0
  48. {runem-0.0.32 → runem-0.1.0}/tests/__init__.py +0 -0
  49. {runem-0.0.32 → runem-0.1.0}/tests/cli/test_initialise_options.py +0 -0
  50. {runem-0.0.32 → runem-0.1.0}/tests/conftest.py +0 -0
  51. {runem-0.0.32 → runem-0.1.0}/tests/data/help_output.3.10.txt +0 -0
  52. {runem-0.0.32 → runem-0.1.0}/tests/data/help_output.3.11.txt +0 -0
  53. {runem-0.0.32 → runem-0.1.0}/tests/intentional_test_error.py +0 -0
  54. {runem-0.0.32 → runem-0.1.0}/tests/sanitise_reports_footer.py +0 -0
  55. {runem-0.0.32 → runem-0.1.0}/tests/test_base.py +0 -0
  56. {runem-0.0.32 → runem-0.1.0}/tests/test_blocking_print.py +0 -0
  57. {runem-0.0.32 → runem-0.1.0}/tests/test_cli.py +0 -0
  58. {runem-0.0.32 → runem-0.1.0}/tests/test_config.py +0 -0
  59. {runem-0.0.32 → runem-0.1.0}/tests/test_config_parse.py +0 -0
  60. {runem-0.0.32 → runem-0.1.0}/tests/test_files.py +0 -0
  61. {runem-0.0.32 → runem-0.1.0}/tests/test_hook_manager.py +0 -0
  62. {runem-0.0.32 → runem-0.1.0}/tests/test_informative_dict.py +0 -0
  63. {runem-0.0.32 → runem-0.1.0}/tests/test_job.py +0 -0
  64. {runem-0.0.32 → runem-0.1.0}/tests/test_job_execute.py +0 -0
  65. {runem-0.0.32 → runem-0.1.0}/tests/test_job_filter.py +0 -0
  66. {runem-0.0.32 → runem-0.1.0}/tests/test_job_runner_simple_command.py +0 -0
  67. {runem-0.0.32 → runem-0.1.0}/tests/test_job_wrapper.py +0 -0
  68. {runem-0.0.32 → runem-0.1.0}/tests/test_job_wrapper_python.py +0 -0
  69. {runem-0.0.32 → runem-0.1.0}/tests/test_report.py +0 -0
  70. {runem-0.0.32 → runem-0.1.0}/tests/test_run_command.py +0 -0
@@ -4,6 +4,73 @@ Changelog
4
4
 
5
5
  (unreleased)
6
6
  ------------
7
+ - Merge pull request #57 from lursight/feat/replace_halo_with_rich.
8
+ [Frank Harrison]
9
+
10
+ feat(rich): moves to using rich instead of halo for spinners
11
+ - Feat(rich): moves to using rich instead of halo for spinners. [Frank
12
+ Harrison]
13
+ - Merge pull request #56 from lursight/feat/better_job_function_typing.
14
+ [Frank Harrison]
15
+
16
+ feat(better-job-function-typing): adds stronger typing for job and hook tasks
17
+ - Feat(better-job-function-typing): adds stronger typing for job and
18
+ hook tasks. [Frank Harrison]
19
+
20
+ We do this by typing the kwargs convenience variable, for which we need
21
+ to use `Unpack` (for back compatibility), and `type_extensions` for
22
+ cross-python-version compatibility (i.e. something that work for all
23
+ targeted version of python).
24
+
25
+ As a side-effect of this we get the benefit of seeing when and where we
26
+ add extra data into the call-stack for `job_execute()` and it emerges
27
+ that we only extend the key-word args when we are calling hooks in a
28
+ non-threaded way, which makes sense.
29
+
30
+ We do several things to achieve this:
31
+ 1. We have common parameters passed to both hooks and job-tasks
32
+ - These common parameters make the hooks and job-task feel similar
33
+ to develop.
34
+ 2. We put all hook-specific kwargs in one place, and mark each as
35
+ optional.
36
+ - this, for now, is mainly because we only have one hook, so this
37
+ will likely change.
38
+ 3. We share and combine kwargs in a range of inheritance types, mainly
39
+ to work with(/around?) python-typing, which isn't great in this
40
+ type of situation.
41
+ - Chore(ignores): adds the tox/ dir to the ignores. [Frank Harrison]
42
+ - Chore(ignores): update git ignore for coverage files and docs gen.
43
+ [Frank Harrison]
44
+
45
+ The docs are recent additions.
46
+ - Merge pull request #55 from lursight/chore/test_improvements. [Frank
47
+ Harrison]
48
+
49
+ Chore/test improvements
50
+ - Chore(coverage): reduce false-positives by deleteing old
51
+ coverage_report files. [Frank Harrison]
52
+
53
+ Sometimes we would have stale .coverage_report.* files left behind where
54
+ from the multi-distributed pytest runs. These would lead more lines
55
+ being reported as coverage than actual - aka false-positive test passes
56
+ for coverage.
57
+ - Chore(help-tests): fixes the help output width in tests. [Frank
58
+ Harrison]
59
+
60
+ This reduces false-negative test results where we get word-splits in
61
+ directories or between process counters or other dynamic content.
62
+
63
+ This was due to the width of the terminal when the test was being run
64
+ being variant with developer-machine. This uses a fixed-width output,
65
+ reducing, if not stopping issues.
66
+ - Chore(deps): updates black 24.3.0 -> 24.10.0. [Frank Harrison]
67
+ - Chore(deps): updates to latest pytest 8.1.1 -> 8.3.3 + plugins. [Frank
68
+ Harrison]
69
+
70
+
71
+ 0.0.32 (2024-11-17)
72
+ -------------------
73
+ - Release: version 0.0.32 🚀 [Frank Harrison]
7
74
  - Merge pull request #54 from lursight/chore/use_tox_on_release. [Frank
8
75
  Harrison]
9
76
 
@@ -1,17 +1,18 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: runem
3
- Version: 0.0.32
3
+ Version: 0.1.0
4
4
  Summary: Awesome runem created by lursight
5
5
  Home-page: https://github.com/lursight/runem/
6
6
  Author: lursight
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
- Requires-Dist: halo
10
9
  Requires-Dist: packaging
11
10
  Requires-Dist: PyYAML
11
+ Requires-Dist: rich
12
+ Requires-Dist: typing_extensions
12
13
  Provides-Extra: tests
13
- Requires-Dist: black==24.3.0; extra == "tests"
14
- Requires-Dist: coverage==7.4.4; extra == "tests"
14
+ Requires-Dist: black==24.10.0; extra == "tests"
15
+ Requires-Dist: coverage==7.5; extra == "tests"
15
16
  Requires-Dist: docformatter==1.7.5; extra == "tests"
16
17
  Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
17
18
  Requires-Dist: flake8==7.0.0; extra == "tests"
@@ -22,10 +23,10 @@ Requires-Dist: mypy==1.9.0; extra == "tests"
22
23
  Requires-Dist: pydocstyle==6.3.0; extra == "tests"
23
24
  Requires-Dist: pylint==3.1.0; extra == "tests"
24
25
  Requires-Dist: pylama==8.4.1; extra == "tests"
25
- Requires-Dist: pytest-cov==4.1.0; extra == "tests"
26
+ Requires-Dist: pytest-cov==6.0.0; extra == "tests"
26
27
  Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
27
- Requires-Dist: pytest-xdist==3.5.0; extra == "tests"
28
- Requires-Dist: pytest==8.1.1; extra == "tests"
28
+ Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
29
+ Requires-Dist: pytest==8.3.3; extra == "tests"
29
30
  Requires-Dist: setuptools; extra == "tests"
30
31
  Requires-Dist: termplotlib==0.3.9; extra == "tests"
31
32
  Requires-Dist: tox; extra == "tests"
@@ -1,6 +1,6 @@
1
1
  # This requirements are for development and testing only, not for production.
2
- black==24.3.0
3
- coverage==7.4.4
2
+ black==24.10.0
3
+ coverage==7.5
4
4
  docformatter==1.7.5
5
5
  flake8-bugbear==24.2.6
6
6
  flake8==7.0.0
@@ -11,10 +11,10 @@ mypy==1.9.0
11
11
  pydocstyle==6.3.0
12
12
  pylint==3.1.0
13
13
  pylama==8.4.1
14
- pytest-cov==4.1.0
14
+ pytest-cov==6.0.0
15
15
  pytest-profiling==1.7.0
16
- pytest-xdist==3.5.0
17
- pytest==8.1.1
16
+ pytest-xdist==3.6.1
17
+ pytest==8.3.3
18
18
  setuptools
19
19
  termplotlib==0.3.9
20
20
  tox
@@ -0,0 +1,9 @@
1
+ packaging
2
+ PyYAML
3
+
4
+ # For UI Elements
5
+ rich
6
+
7
+ # For pre python3.12 Unpack features we need the `typing_extensions` module to
8
+ # be installed.
9
+ typing_extensions
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -12,6 +12,45 @@ from runem.types import JobNames, OptionConfig, OptionsWritable
12
12
  from runem.utils import printable_set
13
13
 
14
14
 
15
+ class HelpFormatterFixedWidth(argparse.HelpFormatter):
16
+ """This works around test issues via constant width helo output.
17
+
18
+ This ensures that we get more constant for help-text by fixing the width to
19
+ something reasonable.
20
+ """
21
+
22
+ def __init__(self, prog: typing.Any) -> None:
23
+ # Override the default width with a fixed width, for tests.
24
+ super().__init__(
25
+ prog,
26
+ # Pretty wide so we do not get wrapping on directories
27
+ # or process-count output.
28
+ width=1000,
29
+ )
30
+
31
+
32
+ def _get_argparse_help_formatter() -> typing.Any:
33
+ """Returns a help-formatter for argparse.
34
+
35
+ This is for tests only to fake terminals of a constant with when rendering help
36
+ output.
37
+ """
38
+ # Check environment variable to see if we're in tests and need a fixed width
39
+ # help output.
40
+ use_fixed_width = os.getenv("RUNEM_FIXED_HELP_WIDTH", None)
41
+
42
+ if use_fixed_width:
43
+ # Use custom formatter with the width specified in the environment variable
44
+ return (
45
+ lambda prog: HelpFormatterFixedWidth( # pylint: disable=unnecessary-lambda
46
+ prog
47
+ )
48
+ )
49
+
50
+ # Use default formatter
51
+ return argparse.HelpFormatter
52
+
53
+
15
54
  def parse_args(
16
55
  config_metadata: ConfigMetadata, argv: typing.List[str]
17
56
  ) -> ConfigMetadata:
@@ -23,7 +62,9 @@ def parse_args(
23
62
  Returns the parsed args, the jobs_names_to_run, job_phases_to_run, job_tags_to_run
24
63
  """
25
64
  parser = argparse.ArgumentParser(
26
- add_help=False, description="Runs the Lursight Lang test-suite"
65
+ add_help=False,
66
+ description="Runs the Lursight Lang test-suite",
67
+ formatter_class=_get_argparse_help_formatter(),
27
68
  )
28
69
  parser.add_argument(
29
70
  "-H", "--help", action="help", help="show this help message and exit"
@@ -1,6 +1,8 @@
1
1
  import typing
2
2
  from collections import defaultdict
3
3
 
4
+ from typing_extensions import Unpack
5
+
4
6
  from runem.config_metadata import ConfigMetadata
5
7
  from runem.job import Job
6
8
  from runem.job_execute import job_execute
@@ -10,6 +12,7 @@ from runem.types import (
10
12
  HookConfig,
11
13
  HookName,
12
14
  Hooks,
15
+ HookSpecificKwargs,
13
16
  HooksStore,
14
17
  JobConfig,
15
18
  )
@@ -69,7 +72,7 @@ class HookManager:
69
72
  self,
70
73
  hook_name: HookName,
71
74
  config_metadata: ConfigMetadata,
72
- **kwargs: typing.Any,
75
+ **kwargs: Unpack[HookSpecificKwargs],
73
76
  ) -> None:
74
77
  """Invokes all functions registered to a specific hook."""
75
78
  hooks: typing.List[HookConfig] = self.hooks_store.get(hook_name, [])
@@ -5,13 +5,17 @@ import uuid
5
5
  from datetime import timedelta
6
6
  from timeit import default_timer as timer
7
7
 
8
+ from typing_extensions import Unpack
9
+
8
10
  from runem.config_metadata import ConfigMetadata
9
11
  from runem.informative_dict import ReadOnlyInformativeDict
10
12
  from runem.job import Job
11
13
  from runem.job_wrapper import get_job_wrapper
12
14
  from runem.log import error, log
13
15
  from runem.types import (
16
+ FilePathList,
14
17
  FilePathListLookup,
18
+ HookSpecificKwargs,
15
19
  JobConfig,
16
20
  JobFunction,
17
21
  JobReturn,
@@ -26,7 +30,7 @@ def job_execute_inner(
26
30
  job_config: JobConfig,
27
31
  config_metadata: ConfigMetadata,
28
32
  file_lists: FilePathListLookup,
29
- **kwargs: typing.Any,
33
+ **kwargs: Unpack[HookSpecificKwargs],
30
34
  ) -> typing.Tuple[JobTiming, JobReturn]:
31
35
  """Wrapper for running a job inside a sub-process.
32
36
 
@@ -36,13 +40,12 @@ def job_execute_inner(
36
40
  if config_metadata.args.verbose:
37
41
  log(f"START: '{label}'")
38
42
  root_path: pathlib.Path = config_metadata.cfg_filepath.parent
39
- function: JobFunction
40
43
  job_tags: typing.Optional[JobTags] = Job.get_job_tags(job_config)
41
44
  os.chdir(root_path)
42
- function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
45
+ function: JobFunction = get_job_wrapper(job_config, config_metadata.cfg_filepath)
43
46
 
44
47
  # get the files for all files found for this job's tags
45
- file_list = Job.get_job_files(file_lists, job_tags)
48
+ file_list: FilePathList = Job.get_job_files(file_lists, job_tags)
46
49
 
47
50
  if not file_list:
48
51
  # no files to work on
@@ -77,8 +80,9 @@ def job_execute_inner(
77
80
  log(f"job: running: '{Job.get_job_name(job_config)}'")
78
81
  reports: JobReturn
79
82
  try:
83
+ assert isinstance(function, JobFunction)
80
84
  reports = function(
81
- options=ReadOnlyInformativeDict(config_metadata.options), # type: ignore
85
+ options=ReadOnlyInformativeDict(config_metadata.options),
82
86
  file_list=file_list,
83
87
  procs=config_metadata.args.procs,
84
88
  root_path=root_path,
@@ -109,7 +113,7 @@ def job_execute(
109
113
  running_jobs: typing.Dict[str, str],
110
114
  config_metadata: ConfigMetadata,
111
115
  file_lists: FilePathListLookup,
112
- **kwargs: typing.Any,
116
+ **kwargs: Unpack[HookSpecificKwargs],
113
117
  ) -> typing.Tuple[JobTiming, JobReturn]:
114
118
  """Thin-wrapper around job_execute_inner needed for mocking in tests.
115
119
 
@@ -18,7 +18,7 @@ def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobF
18
18
  # validate that the command is "understandable" and usable.
19
19
  command_string: str = job_wrapper["command"]
20
20
  validate_simple_command(command_string)
21
- return job_runner_simple_command # type: ignore # NO_COMMIT
21
+ return job_runner_simple_command
22
22
 
23
23
  # if we do not have a simple command address assume we have just an addressed
24
24
  # function
@@ -30,8 +30,11 @@ from datetime import timedelta
30
30
  from itertools import repeat
31
31
  from multiprocessing.managers import DictProxy, ListProxy, ValueProxy
32
32
  from timeit import default_timer as timer
33
+ from types import TracebackType
33
34
 
34
- from halo import Halo
35
+ from rich.console import Console, ConsoleOptions, ConsoleRenderable, RenderResult
36
+ from rich.spinner import Spinner
37
+ from rich.text import Text
35
38
 
36
39
  from runem.command_line import parse_args
37
40
  from runem.config import load_project_config, load_user_configs
@@ -58,6 +61,8 @@ from runem.types import (
58
61
  )
59
62
  from runem.utils import printable_set
60
63
 
64
+ rich_console = Console()
65
+
61
66
 
62
67
  def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
63
68
  """Loads config, parsing cli input and produces the run config.
@@ -85,6 +90,36 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
85
90
  return config_metadata
86
91
 
87
92
 
93
+ class DummySpinner(ConsoleRenderable): # pragma: no cover
94
+ """A dummy spinner for when spinners are disabled."""
95
+
96
+ def __init__(self) -> None:
97
+ self.text = ""
98
+
99
+ def __rich__(self) -> Text:
100
+ """Return a rich Text object for rendering."""
101
+ return Text(self.text)
102
+
103
+ def __rich_console__(
104
+ self, console: Console, options: ConsoleOptions
105
+ ) -> RenderResult:
106
+ """Yield an empty string or placeholder text."""
107
+ yield Text(self.text)
108
+
109
+ def __enter__(self) -> None:
110
+ """Support for context manager."""
111
+ pass
112
+
113
+ def __exit__(
114
+ self,
115
+ exc_type: typing.Optional[typing.Type[BaseException]],
116
+ exc_value: typing.Optional[BaseException],
117
+ traceback: typing.Optional[TracebackType],
118
+ ) -> None:
119
+ """Support for context manager."""
120
+ pass
121
+
122
+
88
123
  def _update_progress(
89
124
  label: str,
90
125
  running_jobs: typing.Dict[str, str],
@@ -104,52 +139,55 @@ def _update_progress(
104
139
  is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
105
140
  num_workers (int): Indicates the number of workers performing the jobs.
106
141
  """
107
- # Using Halo library to show a loading spinner on console
142
+ # Using the `rich` module to show a loading spinner on console
143
+ spinner: typing.Union[Spinner, DummySpinner]
108
144
  if show_spinner:
109
- spinner = Halo(text="", spinner="dots")
110
- spinner.start()
145
+ spinner = Spinner("dots", text="Starting tasks...")
146
+ else:
147
+ spinner = DummySpinner()
111
148
 
112
- # The set of all job labels, and the set of completed jobs
113
- all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
114
- completed_jobs: typing.Set[str] = set()
149
+ with rich_console.status(spinner):
115
150
 
116
- # This dataset is used to track changes between iterations
117
- last_running_jobs_set: typing.Set[str] = set()
151
+ # The set of all job labels, and the set of completed jobs
152
+ all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
153
+ completed_jobs: typing.Set[str] = set()
118
154
 
119
- while is_running.value:
120
- running_jobs_set: typing.Set[str] = set(running_jobs.values())
121
- seen_jobs = list(running_jobs_set.union(seen_jobs)) # Update the seen jobs
155
+ # This dataset is used to track changes between iterations
156
+ last_running_jobs_set: typing.Set[str] = set()
122
157
 
123
- # Jobs that have disappeared since last check
124
- disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
158
+ while is_running.value:
159
+ running_jobs_set: typing.Set[str] = set(running_jobs.values())
160
+ seen_jobs = list(running_jobs_set.union(seen_jobs)) # Update the seen jobs
125
161
 
126
- # Jobs that have not yet completed
127
- remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
162
+ # Jobs that have disappeared since last check
163
+ disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
128
164
 
129
- # Check if we're closing to completion
130
- workers_retiring: bool = len(remaining_jobs) <= num_workers
165
+ # Jobs that have not yet completed
166
+ remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
131
167
 
132
- if workers_retiring:
133
- # Handle edge case: a task may have disappeared whilst process was sleeping
134
- all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
135
- disappeared_jobs.update(all_completed_jobs - running_jobs_set)
168
+ # Check if we're closing to completion
169
+ workers_retiring: bool = len(remaining_jobs) <= num_workers
136
170
 
137
- completed_jobs.update(disappeared_jobs)
171
+ if workers_retiring:
172
+ # Handle edge case: a task may have disappeared whilst process was sleeping
173
+ all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
174
+ disappeared_jobs.update(all_completed_jobs - running_jobs_set)
138
175
 
139
- # Prepare progress report
140
- progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
141
- running_jobs_list = printable_set(running_jobs_set)
142
- if show_spinner:
143
- spinner.text = f"{label}: {progress}({num_workers}): {running_jobs_list}"
176
+ completed_jobs.update(disappeared_jobs)
144
177
 
145
- # Update the tracked dataset for the next iteration
146
- last_running_jobs_set = running_jobs_set
178
+ # Prepare progress report
179
+ progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
180
+ running_jobs_list = printable_set(running_jobs_set)
181
+ if show_spinner:
182
+ spinner.text = (
183
+ f"{label}: {progress}({num_workers}): {running_jobs_list}"
184
+ )
147
185
 
148
- # Sleep to decrease frequency of updates and reduce CPU usage
149
- time.sleep(0.1)
186
+ # Update the tracked dataset for the next iteration
187
+ last_running_jobs_set = running_jobs_set
150
188
 
151
- if show_spinner:
152
- spinner.stop()
189
+ # Sleep to decrease frequency of updates and reduce CPU usage
190
+ time.sleep(0.1)
153
191
 
154
192
 
155
193
  def _process_jobs(
@@ -209,7 +247,7 @@ def _process_jobs(
209
247
  with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
210
248
  # use starmap so we can pass down the job-configs and the args and the files
211
249
  in_out_job_run_metadatas[phase] = pool.starmap(
212
- job_execute,
250
+ job_execute, # no kwargs passed for jobs here
213
251
  zip(
214
252
  jobs,
215
253
  repeat(running_jobs),
@@ -1,9 +1,37 @@
1
- import argparse
1
+ """
2
+
3
+ Some note on Unpack and kwargs:
4
+ We *try* to strongly type `**kwargs` for clarity.
5
+ We have tried several ways to define a Generic type that encapsulates
6
+ `**kwargs: SingleType`
7
+ ... but none of the solutions worked with python 3.9 -> 3.12 and mypy 1.9.0,
8
+ so we have to recommend instead using:
9
+ `**kwargs: Unpack[KwArgsType]`
10
+
11
+ For this to work across versions of python where support for Unpack changes;
12
+ for example `Unpack` is a python 3.12 feature, but available in the
13
+ `typing_extensions` module.
14
+
15
+ So, for now, it looks like we get away with importing `Unpack` from the
16
+ `typing_extensions` module, even in python 3.12, so we will use, and
17
+ recommend using, the `typing_extensions` of `Unpack`, until it becomes
18
+ obsolete.
19
+
20
+ Alternatively, we can use the following, but it's unnecessarily verbose.
21
+
22
+ if sys.version_info >= (3, 12): # pragma: no coverage
23
+ from typing import Unpack
24
+ else: # pragma: no coverage
25
+ from typing_extensions import Unpack
26
+ """
27
+
2
28
  import pathlib
3
29
  import typing
4
30
  from datetime import timedelta
5
31
  from enum import Enum
6
32
 
33
+ from typing_extensions import Unpack
34
+
7
35
  from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
8
36
 
9
37
 
@@ -96,12 +124,6 @@ FilePathSerialise = str
96
124
  FilePathList = typing.List[FilePathSerialise]
97
125
  FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]
98
126
 
99
- # FIXME: this type is no-longer the actual spec of the test-functions
100
- JobFunction = typing.Union[
101
- typing.Callable[[argparse.Namespace, OptionsWritable, FilePathList], None],
102
- typing.Callable[[typing.Any], None],
103
- ]
104
-
105
127
 
106
128
  class JobParamConfig(typing.TypedDict):
107
129
  """Configures what parameters are passed to the test-callable.
@@ -250,3 +272,86 @@ ConfigNodes = typing.Union[
250
272
  Config = typing.List[ConfigNodes]
251
273
 
252
274
  UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
275
+
276
+
277
+ class CommonKwargs(
278
+ typing.TypedDict,
279
+ total=True, # each of these are guaranteed to exist in jobs and hooks
280
+ ):
281
+ """Defines the base args that are passed to all jobs.
282
+
283
+ As we call hooks and job-task in the same manner, this defines the variables that we
284
+ can access from both hooks and job-tasks.
285
+ """
286
+
287
+ root_path: pathlib.Path # the path where the .runem.yml file is
288
+ job: JobConfig # the job or hook task spec ¢ TODO: rename this
289
+ label: str # the name of the hook or the job-label
290
+ options: Options # options passed in on the command line
291
+ procs: int # the max number of concurrent procs to run
292
+ verbose: bool # control log verbosity
293
+
294
+
295
+ class HookSpecificKwargs(typing.TypedDict, total=False):
296
+ """Defines the args that are passed down to the hooks.
297
+
298
+ NOTE: that although these however
299
+ outside of the *hook* context, the data will not be present. Such is the
300
+ difficulty in dynamic programming.
301
+ """
302
+
303
+ wall_clock_time_saved: timedelta # only on `HookName.ON_EXIT`
304
+
305
+
306
+ class JobTaskKwargs(
307
+ typing.TypedDict,
308
+ total=False, # for now, we don't enforce these types for job-context, but we should.
309
+ ):
310
+ """Defines the task-specific args for job-task functions."""
311
+
312
+ file_list: FilePathList
313
+ record_sub_job_time: typing.Optional[typing.Callable[[str, timedelta], None]]
314
+
315
+
316
+ class HookKwargs(CommonKwargs, HookSpecificKwargs):
317
+ """A merged set of kwargs for runem-hooks."""
318
+
319
+ pass
320
+
321
+
322
+ class JobKwargs(CommonKwargs, JobTaskKwargs):
323
+ """A merged set of kwargs for job-tasks."""
324
+
325
+ pass
326
+
327
+
328
+ class AllKwargs(CommonKwargs, JobTaskKwargs, HookSpecificKwargs):
329
+ """A merged set of kwargs for al job-functions."""
330
+
331
+ pass
332
+
333
+
334
+ @typing.runtime_checkable
335
+ class JobFunction(typing.Protocol):
336
+ def __call__(self, **kwargs: Unpack[AllKwargs]) -> JobReturn: # pragma: no cover
337
+ """Defines the call() protocol's abstract pattern for job-tasks."""
338
+
339
+ @property
340
+ def __name__(self) -> str: # pragma: no cover
341
+ """Defines the name protocol for job-task functions.
342
+
343
+ This is primarily used for internal tests but can be useful for introspection.
344
+ """
345
+
346
+
347
+ def _hook_example(
348
+ wall_clock_time_saved: timedelta,
349
+ **kwargs: typing.Any,
350
+ ) -> None:
351
+ """An example hook."""
352
+
353
+
354
+ def _job_task_example(
355
+ **kwargs: Unpack[JobKwargs],
356
+ ) -> None:
357
+ """An example job-task function."""
@@ -1,17 +1,18 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: runem
3
- Version: 0.0.32
3
+ Version: 0.1.0
4
4
  Summary: Awesome runem created by lursight
5
5
  Home-page: https://github.com/lursight/runem/
6
6
  Author: lursight
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
- Requires-Dist: halo
10
9
  Requires-Dist: packaging
11
10
  Requires-Dist: PyYAML
11
+ Requires-Dist: rich
12
+ Requires-Dist: typing_extensions
12
13
  Provides-Extra: tests
13
- Requires-Dist: black==24.3.0; extra == "tests"
14
- Requires-Dist: coverage==7.4.4; extra == "tests"
14
+ Requires-Dist: black==24.10.0; extra == "tests"
15
+ Requires-Dist: coverage==7.5; extra == "tests"
15
16
  Requires-Dist: docformatter==1.7.5; extra == "tests"
16
17
  Requires-Dist: flake8-bugbear==24.2.6; extra == "tests"
17
18
  Requires-Dist: flake8==7.0.0; extra == "tests"
@@ -22,10 +23,10 @@ Requires-Dist: mypy==1.9.0; extra == "tests"
22
23
  Requires-Dist: pydocstyle==6.3.0; extra == "tests"
23
24
  Requires-Dist: pylint==3.1.0; extra == "tests"
24
25
  Requires-Dist: pylama==8.4.1; extra == "tests"
25
- Requires-Dist: pytest-cov==4.1.0; extra == "tests"
26
+ Requires-Dist: pytest-cov==6.0.0; extra == "tests"
26
27
  Requires-Dist: pytest-profiling==1.7.0; extra == "tests"
27
- Requires-Dist: pytest-xdist==3.5.0; extra == "tests"
28
- Requires-Dist: pytest==8.1.1; extra == "tests"
28
+ Requires-Dist: pytest-xdist==3.6.1; extra == "tests"
29
+ Requires-Dist: pytest==8.3.3; extra == "tests"
29
30
  Requires-Dist: setuptools; extra == "tests"
30
31
  Requires-Dist: termplotlib==0.3.9; extra == "tests"
31
32
  Requires-Dist: tox; extra == "tests"
@@ -1,10 +1,11 @@
1
- halo
2
1
  packaging
3
2
  PyYAML
3
+ rich
4
+ typing_extensions
4
5
 
5
6
  [tests]
6
- black==24.3.0
7
- coverage==7.4.4
7
+ black==24.10.0
8
+ coverage==7.5
8
9
  docformatter==1.7.5
9
10
  flake8-bugbear==24.2.6
10
11
  flake8==7.0.0
@@ -15,10 +16,10 @@ mypy==1.9.0
15
16
  pydocstyle==6.3.0
16
17
  pylint==3.1.0
17
18
  pylama==8.4.1
18
- pytest-cov==4.1.0
19
+ pytest-cov==6.0.0
19
20
  pytest-profiling==1.7.0
20
- pytest-xdist==3.5.0
21
- pytest==8.1.1
21
+ pytest-xdist==3.6.1
22
+ pytest==8.3.3
22
23
  setuptools
23
24
  termplotlib==0.3.9
24
25
  tox
@@ -669,6 +669,8 @@ def test_runem_help(
669
669
  As we build features we want to ensure that the help output stays consistent as we
670
670
  leverage the argparse system to generate the help for a specific .runem.yml config
671
671
  """
672
+ # Ensure we get fixed-width output.
673
+ os.environ["RUNEM_FIXED_HELP_WIDTH"] = "1"
672
674
  runem_cli_switches: typing.List[str] = ["--help"]
673
675
  runem_stdout: typing.List[str]
674
676
  error_raised: typing.Optional[BaseException]
@@ -1,3 +0,0 @@
1
- halo
2
- packaging
3
- PyYAML
@@ -1 +0,0 @@
1
- 0.0.32
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes