runem 0.0.17__py3-none-any.whl → 0.0.18__py3-none-any.whl

runem/command_line.py

@@ -147,7 +147,7 @@ def parse_args(
 
     args = parser.parse_args(argv[1:])
 
-    options: Options = _initialise_options(config_metadata, args)
+    options: Options = initialise_options(config_metadata, args)
 
     if not _validate_filters(config_metadata, args):
         sys.exit(1)
@@ -214,7 +214,7 @@ def _validate_filters(
     return True
 
 
-def _initialise_options(
+def initialise_options(
     config_metadata: ConfigMetadata,
     args: argparse.Namespace,
 ) -> Options:
@@ -227,7 +227,7 @@ def _initialise_options(
         option["name"]: option["default"] for option in config_metadata.options_config
     }
     if config_metadata.options_config and args.overrides_on:  # pragma: no branch
-        for option_name in args.overrides_on:
+        for option_name in args.overrides_on:  # pragma: no branch
             options[option_name] = True
     if config_metadata.options_config and args.overrides_off:  # pragma: no branch
         for option_name in args.overrides_off:

runem/config_parse.py

@@ -4,7 +4,7 @@ import typing
 from collections import defaultdict
 
 from runem.config_metadata import ConfigMetadata
-from runem.job_function_python import get_job_function
+from runem.job_wrapper_python import get_job_wrapper
 from runem.log import log
 from runem.types import (
     Config,
@@ -79,7 +79,7 @@ def parse_job_config(
             sys.exit(1)
 
         # try and load the function _before_ we schedule it's execution
-        get_job_function(job, cfg_filepath)
+        get_job_wrapper(job, cfg_filepath)
         phase_id: PhaseName = job["when"]["phase"]
         in_out_jobs_by_phase[phase_id].append(job)
 

runem/{job_runner.py → job_execute.py}

@@ -2,16 +2,17 @@ import inspect
 import os
 import pathlib
 import typing
+import uuid
 from datetime import timedelta
 from timeit import default_timer as timer
 
 from runem.config_metadata import ConfigMetadata
-from runem.job_function_python import get_job_function
+from runem.job_wrapper_python import get_job_wrapper
 from runem.log import log
 from runem.types import FilePathList, FilePathListLookup, JobConfig, JobReturn, JobTags
 
 
-def job_runner_inner(
+def job_execute_inner(
     job_config: JobConfig,
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
@@ -27,7 +28,7 @@ def job_runner_inner(
     function: typing.Callable
     job_tags: JobTags = set(job_config["when"]["tags"])
     os.chdir(root_path)
-    function = get_job_function(job_config, config_metadata.cfg_filepath)
+    function = get_job_wrapper(job_config, config_metadata.cfg_filepath)
 
     # get the files for all files found for this job's tags
     file_list: FilePathList = []
@@ -74,13 +75,18 @@ def job_runner_inner(
     return (timing_data, reports)
 
 
-def job_runner(
+def job_execute(
     job_config: JobConfig,
+    running_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
 ) -> typing.Tuple[typing.Tuple[str, timedelta], JobReturn]:
-    """Thing wrapper around job_runner_inner needed fro mocking in tests.
+    """Thin-wrapper around job_execute_inner needed for mocking in tests.
 
     Needed for faster tests.
     """
-    return job_runner_inner(job_config, config_metadata, file_lists)
+    this_id: str = str(uuid.uuid4())
+    running_jobs[this_id] = job_config["label"]
+    results = job_execute_inner(job_config, config_metadata, file_lists)
+    del running_jobs[this_id]
+    return results

runem/job_filter.py

@@ -77,10 +77,14 @@ def filter_jobs(
         log(f"filtering for tags {printable_set(tags_to_run)}", decorate=True, end="")
     if tags_to_avoid:
         if tags_to_run:
-            log(", ", end="")
+            log(", ", decorate=False, end="")
         else:
-            log(decorate=True)
-        log(f"excluding jobs with tags {printable_set(tags_to_avoid)}", end="")
+            log(decorate=True, end="")
+        log(
+            f"excluding jobs with tags {printable_set(tags_to_avoid)}",
+            decorate=False,
+            end="",
+        )
     if tags_to_run or tags_to_avoid:
         log(decorate=False)
     filtered_jobs: PhaseGroupedJobs = defaultdict(list)

runem/{job_function_python.py → job_wrapper_python.py}

@@ -20,20 +20,35 @@ def _load_python_function_from_module(
     ).absolute()
 
     # load the function
-    module_spec = module_spec_from_file_location(function_to_load, abs_module_file_path)
-    if not module_spec:
+    try:
+        module_spec = module_spec_from_file_location(
+            function_to_load, abs_module_file_path
+        )
+        if not module_spec:
+            raise FileNotFoundError()
+        if not module_spec.loader:
+            raise FunctionNotFound("unable to load module")
+    except FileNotFoundError as err:
         raise FunctionNotFound(
             (
                 f"unable to load '{function_to_load}' from '{str(module_file_path)} "
                 f"relative to '{str(cfg_filepath)}"
             )
-        )
+        ) from err
 
     module = module_from_spec(module_spec)
-    sys.modules[module_name] = module
-    if not module_spec.loader:
+    if not module:
         raise FunctionNotFound("unable to load module")
-    module_spec.loader.exec_module(module)
+    sys.modules[module_name] = module
+    try:
+        module_spec.loader.exec_module(module)
+    except FileNotFoundError as err:
+        raise FunctionNotFound(
+            (
+                f"unable to load '{function_to_load}' from '{str(module_file_path)} "
+                f"relative to '{str(cfg_filepath)}"
+            )
+        ) from err
     try:
         function: JobFunction = getattr(module, function_to_load)
     except AttributeError as err:
@@ -70,7 +85,7 @@ def _find_job_module(cfg_filepath: pathlib.Path, module_file_path: str) -> pathl
     return module_path.relative_to(cfg_filepath.parent.absolute())
 
 
-def get_job_function(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
+def get_job_wrapper(job_config: JobConfig, cfg_filepath: pathlib.Path) -> JobFunction:
     """Given a job-description dynamically loads the job-function so we can call it.
 
     Side-effects: also re-addressed the job-config.

runem/log.py

@@ -1,8 +1,11 @@
+import typing
 
-def log(msg:str="", decorate:bool=True, end: str|None=None)->None: 
-    """thin wrapper around 'print', so we can change the output
-    
-    One way we change it is to decorate the output with 'runem'"""
+
+def log(msg: str = "", decorate: bool = True, end: typing.Optional[str] = None) -> None:
+    """Thin wrapper around 'print', so we can change the output.
+
+    One way we change it is to decorate the output with 'runem'
+    """
     if decorate:
         msg = f"runem: {msg}"
-    print(msg, end=end)
+    print(msg, end=end)

runem/report.py

@@ -26,7 +26,10 @@ def _plot_times(
     phase_run_oder: OrderedPhases,
     timing_data: JobRunTimesByPhase,
 ) -> timedelta:
-    """Prints a report to terminal on how well we performed."""
+    """Prints a report to terminal on how well we performed.
+
+    Also calculates the wall-clock time-saved for the user.
+    """
     labels: typing.List[str] = []
     times: typing.List[float] = []
     job_time_sum: timedelta = timedelta()  # init to 0
@@ -66,12 +69,31 @@ def _plot_times(
     return time_saved
 
 
+def _print_reports_by_phase(
+    phase_run_oder: OrderedPhases, report_data: JobRunReportByPhase
+) -> None:
+    """Logs out the reports by grouped by phase."""
+    for phase in phase_run_oder:
+        report_urls: ReportUrls = report_data[phase]
+        job_report_url_info: ReportUrlInfo
+        for job_report_url_info in report_urls:
+            if not job_report_url_info:
+                continue
+            log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
+
+
 def report_on_run(
     phase_run_oder: OrderedPhases,
     job_run_metadatas: JobRunMetadatasByPhase,
     overall_runtime: timedelta,
-):
+) -> timedelta:
+    """Generate high-level reports AND prints out any reports returned by jobs.
+
+    IMPORTANT: returns the wall-clock time saved to the user.
+    """
     log("reports:")
+
+    # First, collate all data, timing and reports
     timing_data: JobRunTimesByPhase = defaultdict(list)
     report_data: JobRunReportByPhase = defaultdict(list)
     phase: PhaseName
@@ -81,17 +103,21 @@ def report_on_run(
         for timing, reports in job_run_metadatas[phase]:
             timing_data[phase].append(timing)
             if reports:
+                # the job returned some report urls, record them against the
+                # job's phase
                 report_data[phase].extend(reports["reportUrls"])
+
+    # Now plot the times on the terminal to give a visual report of the timing.
+    # Also, calculate the time saved by runem, a key selling-point metric
     time_saved: timedelta = _plot_times(
         overall_run_time=overall_runtime,
         phase_run_oder=phase_run_oder,
         timing_data=timing_data,
     )
-    for phase in phase_run_oder:
-        report_urls: ReportUrls = report_data[phase]
-        job_report_url_info: ReportUrlInfo
-        for job_report_url_info in report_urls:
-            if not job_report_url_info:
-                continue
-            log(f"report: {str(job_report_url_info[0])}: {str(job_report_url_info[1])}")
+
+    # Penultimate-ly print out the available reports grouped by run-phase.
+    _print_reports_by_phase(phase_run_oder, report_data)
+
+    # Return the key metric for runem, the wall-clock time saved to the user
+    # TODO: write this to disk
     return time_saved

runem/runem.py

@@ -23,19 +23,23 @@ import multiprocessing
 import os
 import pathlib
 import sys
+import time
 import typing
 from collections import defaultdict
 from datetime import timedelta
 from itertools import repeat
+from multiprocessing.managers import DictProxy, ValueProxy
 from timeit import default_timer as timer
 
+from halo import Halo
+
 from runem.command_line import parse_args
 from runem.config import load_config
 from runem.config_metadata import ConfigMetadata
 from runem.config_parse import parse_config
 from runem.files import find_files
+from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
-from runem.job_runner import job_runner
 from runem.log import log
 from runem.report import report_on_run
 from runem.types import (
@@ -75,6 +79,22 @@ def _determine_run_parameters(argv: typing.List[str]) -> ConfigMetadata:
     return config_metadata
 
 
+def _progress_updater(
+    label: str, running_jobs: typing.Dict[str, str], is_running: ValueProxy
+) -> None:
+    spinner = Halo(text="", spinner="dots")
+    spinner.start()
+
+    while is_running.value:
+        running_job_names: typing.List[str] = [
+            f"'{job}'" for job in sorted(list(running_jobs.values()))
+        ]
+        printable_jobs: str = ", ".join(running_job_names)
+        spinner.text = f"{label}: {printable_jobs}"
+        time.sleep(0.1)
+    spinner.stop()
+
+
 def _process_jobs(
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
@@ -91,28 +111,44 @@ def _process_jobs(
           it and, for instance, run the longest-running job first with quicker
           jobs completing around it, then we would work out that schedule here.
     """
-    max_num_concurrent_procs: int = (
+    num_concurrent_procs: int = (
         config_metadata.args.procs
         if config_metadata.args.procs != -1
         else multiprocessing.cpu_count()
     )
-    num_concurrent_procs:int = min(max_num_concurrent_procs, len(jobs))
+    num_concurrent_procs = min(num_concurrent_procs, len(jobs))
     log(
         (
-            f"Running '{phase}' with {num_concurrent_procs} workers (of "
-            f"{max_num_concurrent_procs} max) processing {len(jobs)} jobs"
+            f"Running '{phase}' with {num_concurrent_procs} workers "
+            f"processing {len(jobs)} jobs"
         )
     )
-    with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
-        # use starmap so we can pass down the job-configs and the args and the files
-        in_out_job_run_metadatas[phase] = pool.starmap(
-            job_runner,
-            zip(
-                jobs,
-                repeat(config_metadata),
-                repeat(file_lists),
-            ),
+
+    with multiprocessing.Manager() as manager:
+        running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
+        is_running: ValueProxy = manager.Value("b", True)
+
+        terminal_writer_process = multiprocessing.Process(
+            target=_progress_updater, args=(phase, running_jobs, is_running)
         )
+        terminal_writer_process.start()
+
+        try:
+            with multiprocessing.Pool(processes=num_concurrent_procs) as pool:
+                # use starmap so we can pass down the job-configs and the args and the files
+                in_out_job_run_metadatas[phase] = pool.starmap(
+                    job_execute,
+                    zip(
+                        jobs,
+                        repeat(running_jobs),
+                        repeat(config_metadata),
+                        repeat(file_lists),
+                    ),
+                )
+        finally:
+            # Signal the terminal_writer process to exit
+            is_running.value = False
+            terminal_writer_process.join()
 
 
 def _process_jobs_by_phase(

runem/types.py

@@ -18,8 +18,9 @@ JobPhases = typing.Set[str]
 JobTags = typing.Set[JobTag]
 PhaseName = str
 OrderedPhases = typing.Tuple[PhaseName, ...]
-ReportUrl = str | pathlib.Path
-ReportUrlInfo = typing.Tuple[str, ReportUrl]
+ReportName = str
+ReportUrl = typing.Union[str, pathlib.Path]
+ReportUrlInfo = typing.Tuple[ReportName, ReportUrl]
 ReportUrls = typing.List[ReportUrlInfo]
 
 

{runem-0.0.17.dist-info → runem-0.0.18.dist-info}/METADATA

@@ -1,11 +1,12 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.0.17
+Version: 0.0.18
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: halo
 Requires-Dist: PyYAML
 Provides-Extra: test
 Requires-Dist: black ==23.11.0 ; extra == 'test'

runem-0.0.18.dist-info/RECORD

@@ -0,0 +1,25 @@
+runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
+runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
+runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
+runem/command_line.py,sha256=lb1tvYOEVYpB_bUTRL8KxjLFuV4vKn8btFKm-28GMu0,9464
+runem/config.py,sha256=h0r2WYUlAgdokUS1VU1sjFHgrN_cEq339ph4Kn1IBBM,1937
+runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
+runem/config_parse.py,sha256=gso5Lziw8yII5jGSx8fR-IAs7nxUzhzitnRAEXsg5f8,5157
+runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
+runem/job_execute.py,sha256=x-IGTG5KYnTJtCYm4jP8_WOQ_UM_BAiG5b3UUKp527A,3042
+runem/job_filter.py,sha256=nltSRK1IKp2fuTCHOZFcii0FBL-R3RKvrnyWXTPjoMw,3478
+runem/job_wrapper_python.py,sha256=TG9bzDe1dgTnsrXrCPd6LzN8qbMkH4rEv_vXC_zlJc0,4226
+runem/log.py,sha256=m1lI4V8ygM53pY4Go4eEzvEJY8srVoItxUNhdp_Vrqg,314
+runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+runem/report.py,sha256=AyD2yT_8ODo5QRuFkVFIecp6ycT4XElJbleWviZbCfI,4032
+runem/run_command.py,sha256=6QxH8pb4Lug-d0MCP2lBKq2TJlQloxRlq0hYJM6IScc,4528
+runem/runem.py,sha256=Q5ryZtArAGuTlHF07aobJkRIzurKIUsMSUPzEksL6LY,8227
+runem/types.py,sha256=Hw7G7ut7yNaoc3XiDNKeM-cAU-DDu3fDOne13FrutS0,5374
+runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
+runem-0.0.18.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+runem-0.0.18.dist-info/METADATA,sha256=VzkG4gPaopCtwCnEx1XRGLk9iNWpLPjl4x8BY-JvSSA,15909
+runem-0.0.18.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+runem-0.0.18.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
+runem-0.0.18.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
+runem-0.0.18.dist-info/RECORD,,

runem-0.0.17.dist-info/RECORD

@@ -1,25 +0,0 @@
-runem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runem/__main__.py,sha256=dsOiVZegpfK9JOs5n7UmbX5iwwbj7iFkEbLoVeEgAn4,136
-runem/base.py,sha256=EZfR7FIlwEdU9Vfe47Wk2DOO8GQqpKxxLNKp6YHueZ4,316
-runem/cli.py,sha256=Hz9j6Iy1SWjW2uZlQZp0eX48NDWD1-QhIqlBzr_qz0s,125
-runem/command_line.py,sha256=4h5YL_eyDKH472BxMeQi9ONjxlGKpe4rI7Ny09kY2Y0,9445
-runem/config.py,sha256=h0r2WYUlAgdokUS1VU1sjFHgrN_cEq339ph4Kn1IBBM,1937
-runem/config_metadata.py,sha256=EjCEqx9-2mtMrFf3lJJ8HFhfmScioZKeY_c9Rzajne8,2836
-runem/config_parse.py,sha256=s7Ty7i-KwktFg2LdLWoJaxRXnWCCT19hVGYbwgwBLp8,5160
-runem/files.py,sha256=vAI17m-Y1tRGot6_vDpTuBTkKm8xdByGoh7HCfNkMFU,1900
-runem/job_filter.py,sha256=90gY-p29vEb17v8R1CCgBkfQs4oDRTgvBdT9YZBiWSA,3391
-runem/job_function_python.py,sha256=iNRKQp0Rlkh1VektOxUSa_KqrrY0yB7jDChcRt-s6oQ,3754
-runem/job_runner.py,sha256=FUnNinHXQJAT-0OCD4rWJ81rwqsPwGAzuejvr3bHIL0,2852
-runem/log.py,sha256=tzpOW5UOwq6u56v56U4tR1_00FMZIiRYHNg11pG-vv8,275
-runem/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-runem/report.py,sha256=d3wPg_fqefaMTW1EQTU74zo4-qtPiF-MfWPKH3Mokbg,3085
-runem/run_command.py,sha256=6QxH8pb4Lug-d0MCP2lBKq2TJlQloxRlq0hYJM6IScc,4528
-runem/runem.py,sha256=_Xr4tITqpgYAI_Ty2ZrWFtq0_5328840NSxHTTUOiYI,7031
-runem/types.py,sha256=IxUDGHOQZUc86OPjDHDKwrcBFr78v-MNvTpPwUzWoaI,5337
-runem/utils.py,sha256=3N_kel9LsriiMq7kOjT14XhfxUOgz4hdDg97wlLKm3U,221
-runem-0.0.17.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-runem-0.0.17.dist-info/METADATA,sha256=kt13DdIGd0yuQenTfPPKovO0-xz5K7h5LzjlP_H2czc,15889
-runem-0.0.17.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-runem-0.0.17.dist-info/entry_points.txt,sha256=nu0g_vBeuPihYtimbtlNusxWovylMppvJ8UxdJlJfvM,46
-runem-0.0.17.dist-info/top_level.txt,sha256=gK6iqh9OfHDDpErioCC9ul_zx2Q5zWTALtcuGU7Vil4,6
-runem-0.0.17.dist-info/RECORD,,