runem 0.1.0__tar.gz → 0.1.1__tar.gz

{runem-0.1.0 → runem-0.1.1}/HISTORY.md

@@ -4,6 +4,27 @@ Changelog
 
 (unreleased)
 ------------
+- Merge pull request #58 from lursight/fix/completed_job_counts. [Frank
+  Harrison]
+
+  fix(complete-job-count): fixes and simplifies the in-progress report code
+- Fix(complete-job-count): fixes and simplifies the in-progress report
+  code. [Frank Harrison]
+
+  The core problem was that the remaining-job-count didn't match the
+  number of job-labels that were being shown.
+
+  The root-cause was due to some of the job-tasks completing before the
+  `_update_progress()` thread had started monitoring the jobs.
+
+  We fix this by adding a new parameter to the job-executer that tracks
+  when job completed. It's a very simple fix and reduces the overall
+  complexity for about the same cost of threading-primitives.
+
+
+0.1.0 (2024-11-17)
+------------------
+- Release: version 0.1.0 🚀 [Frank Harrison]
 - Merge pull request #57 from lursight/feat/replace_halo_with_rich.
   [Frank Harrison]
 

{runem-0.1.0 → runem-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.1.0
+Version: 0.1.1
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight

runem-0.1.1/runem/VERSION

@@ -0,0 +1 @@
+0.1.1

{runem-0.1.0 → runem-0.1.1}/runem/hook_manager.py

@@ -95,6 +95,7 @@ class HookManager:
             job_execute(
                 job_config,
                 running_jobs={},
+                completed_jobs={},
                 config_metadata=config_metadata,
                 file_lists=file_lists,
                 **kwargs,

{runem-0.1.0 → runem-0.1.1}/runem/job_execute.py

@@ -111,6 +111,7 @@ def job_execute_inner(
 def job_execute(
     job_config: JobConfig,
     running_jobs: typing.Dict[str, str],
+    completed_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
     **kwargs: Unpack[HookSpecificKwargs],
@@ -127,5 +128,6 @@ def job_execute(
         file_lists,
         **kwargs,
     )
+    completed_jobs[this_id] = running_jobs[this_id]
     del running_jobs[this_id]
     return results

{runem-0.1.0 → runem-0.1.1}/runem/runem.py

@@ -28,7 +28,7 @@ import typing
 from collections import defaultdict
 from datetime import timedelta
 from itertools import repeat
-from multiprocessing.managers import DictProxy, ListProxy, ValueProxy
+from multiprocessing.managers import DictProxy, ValueProxy
 from timeit import default_timer as timer
 from types import TracebackType
 
@@ -41,7 +41,6 @@ from runem.config import load_project_config, load_user_configs
 from runem.config_metadata import ConfigMetadata
 from runem.config_parse import load_config_metadata
 from runem.files import find_files
-from runem.job import Job
 from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
 from runem.log import error, log, warn
@@ -123,7 +122,7 @@ class DummySpinner(ConsoleRenderable):  # pragma: no cover
 def _update_progress(
     label: str,
     running_jobs: typing.Dict[str, str],
-    seen_jobs: typing.List[str],
+    completed_jobs: typing.Dict[str, str],
     all_jobs: Jobs,
     is_running: ValueProxy[bool],
     num_workers: int,
@@ -134,7 +133,6 @@ def _update_progress(
     Args:
         label (str): The identifier.
         running_jobs (Dict[str, str]): The currently running jobs.
-        seen_jobs (List[str]): Jobs that the function has previously tracked.
         all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
@@ -146,47 +144,25 @@ def _update_progress(
     else:
         spinner = DummySpinner()
 
-    with rich_console.status(spinner):
-
-        # The set of all job labels, and the set of completed jobs
-        all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
-        completed_jobs: typing.Set[str] = set()
-
-        # This dataset is used to track changes between iterations
-        last_running_jobs_set: typing.Set[str] = set()
+    last_running_jobs_set: typing.Set[str] = set()
 
+    with rich_console.status(spinner):
         while is_running.value:
             running_jobs_set: typing.Set[str] = set(running_jobs.values())
-            seen_jobs = list(running_jobs_set.union(seen_jobs))  # Update the seen jobs
-
-            # Jobs that have disappeared since last check
-            disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
-
-            # Jobs that have not yet completed
-            remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
 
-            # Check if we're closing to completion
-            workers_retiring: bool = len(remaining_jobs) <= num_workers
-
-            if workers_retiring:
-                # Handle edge case: a task may have disappeared whilst process was sleeping
-                all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
-                disappeared_jobs.update(all_completed_jobs - running_jobs_set)
-
-            completed_jobs.update(disappeared_jobs)
-
-            # Prepare progress report
+            # Progress report
             progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
-            running_jobs_list = printable_set(running_jobs_set)
+            running_jobs_list = printable_set(
+                running_jobs_set
+            )  # Reflect current running jobs accurately
+            report: str = f"{label}: {progress}({num_workers}): {running_jobs_list}"
             if show_spinner:
-                spinner.text = (
-                    f"{label}: {progress}({num_workers}): {running_jobs_list}"
-                )
-
-            # Update the tracked dataset for the next iteration
-            last_running_jobs_set = running_jobs_set
+                spinner.text = report
+            else:
+                if last_running_jobs_set != running_jobs_set:
+                    rich_console.log(report)
 
-            # Sleep to decrease frequency of updates and reduce CPU usage
+            # Sleep for reduced CPU usage
             time.sleep(0.1)
 
 
@@ -225,8 +201,8 @@ def _process_jobs(
     subprocess_error: typing.Optional[BaseException] = None
 
     with multiprocessing.Manager() as manager:
-        seen_jobs: ListProxy[str] = manager.list()
         running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
+        completed_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
         is_running: ValueProxy[bool] = manager.Value("b", True)
 
         terminal_writer_process = multiprocessing.Process(
@@ -234,7 +210,7 @@ def _process_jobs(
             args=(
                 phase,
                 running_jobs,
-                seen_jobs,
+                completed_jobs,
                 jobs,
                 is_running,
                 num_concurrent_procs,
@@ -251,6 +227,7 @@ def _process_jobs(
                     zip(
                         jobs,
                         repeat(running_jobs),
+                        repeat(completed_jobs),
                         repeat(config_metadata),
                         repeat(file_lists),
                     ),

{runem-0.1.0 → runem-0.1.1}/runem.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.1.0
+Version: 0.1.1
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight

{runem-0.1.0 → runem-0.1.1}/tests/test_job_execute.py

@@ -45,7 +45,7 @@ def _job_execute_and_capture_stdout(
     ret_err: typing.Optional[BaseException] = None
     with io.StringIO() as buf, redirect_stdout(buf):
         try:
-            job_execute(job_config, running_jobs, config_metadata, file_lists)
+            job_execute(job_config, running_jobs, {}, config_metadata, file_lists)
         except BaseException as err:  # pylint: disable=broad-exception-caught
             # capture the error and return it
             ret_err = err

{runem-0.1.0 → runem-0.1.1}/tests/test_runem.py

@@ -1675,11 +1675,12 @@ def test_progress_updater_with_running_jobs(
     show_spinner: bool,
 ) -> None:
     running_jobs: typing.Dict[str, str] = {"job1": "running", "job2": "pending"}
+    completed_jobs: typing.Dict[str, str] = {}
     with pytest.raises(SleepCalledError), multiprocessing.Manager() as manager:
         _update_progress(
             "dummy label",
             running_jobs,
-            seen_jobs=[],
+            completed_jobs,
             all_jobs=[],
             is_running=manager.Value("b", True),
             num_workers=1,
@@ -1690,6 +1691,7 @@ def test_progress_updater_with_running_jobs(
 
 def test_progress_updater_with_running_jobs_and_10_jobs(mock_sleep: Mock) -> None:
     running_jobs: typing.Dict[str, str] = {"job1": "running", "job2": "pending"}
+    completed_jobs: typing.Dict[str, str] = {}
     job_config: JobConfig = {
         "addr": {
             "file": __file__,
@@ -1715,7 +1717,7 @@ def test_progress_updater_with_running_jobs_and_10_jobs(mock_sleep: Mock) -> Non
         _update_progress(
             "dummy label",
             running_jobs,
-            seen_jobs=[],
+            completed_jobs,
             all_jobs=all_jobs,
             is_running=manager.Value("b", True),
             num_workers=1,
@@ -1726,11 +1728,12 @@ def test_progress_updater_with_running_jobs_and_10_jobs(mock_sleep: Mock) -> Non
 
 def test_progress_updater_without_running_jobs(mock_sleep: Mock) -> None:
     running_jobs: typing.Dict[str, str] = {}
+    completed_jobs: typing.Dict[str, str] = {}
     with pytest.raises(SleepCalledError), multiprocessing.Manager() as manager:
         _update_progress(
             "dummy label",
             running_jobs,
-            seen_jobs=[],
+            completed_jobs,
             all_jobs=[],
             is_running=manager.Value("b", True),
             num_workers=1,
@@ -1741,11 +1744,12 @@ def test_progress_updater_without_running_jobs(mock_sleep: Mock) -> None:
 
 def test_progress_updater_with_empty_running_jobs(mock_sleep: Mock) -> None:
     running_jobs: typing.Dict[str, str] = {"job1": ""}
+    completed_jobs: typing.Dict[str, str] = {}
     with pytest.raises(SleepCalledError), multiprocessing.Manager() as manager:
         _update_progress(
             "dummy label",
             running_jobs,
-            seen_jobs=[],
+            completed_jobs,
             all_jobs=[],
             is_running=manager.Value("b", True),
             num_workers=1,
@@ -1769,7 +1773,7 @@ def test_progress_updater_with_false(show_spinner: bool) -> None:
         _update_progress(
             "dummy label",
             running_jobs,
-            [],
+            {},
             [],
             manager.Value("b", False),
             1,

runem-0.1.0/runem/VERSION

@@ -1 +0,0 @@
-0.1.0