runem 0.1.0__tar.gz → 0.1.2__tar.gz

{runem-0.1.0 → runem-0.1.2}/HISTORY.md

@@ -4,6 +4,77 @@ Changelog
 
 (unreleased)
 ------------
+- Chore(actions): simplifies code paths for actions. [Frank Harrison]
+- Chore(contrib): removes the Makefile commands that have been removed.
+  [Frank Harrison]
+
+  .. this should have been in the last PR.
+- Chore(contrib): update the CONTRIBUTING docs and removes defunct
+  scripts from Makefile. [Frank Harrison]
+- Chore(faster-actions): removes redundant github actions. [Frank
+  Harrison]
+
+  This should speed up github actions, but it may lead to more false-positives as we are relying on runem to be working and realiabel to run our own checks....
+
+  This could come back to bite us.
+- Chore(extra-ctx): adds config_metadata to job kw-args. [Frank
+  Harrison]
+
+  This allows much greater depth of testing based on other jobs-inclusion and metadata.
+- Chore(extra-ctx): moves the job-task calling to use kwargs explicitly.
+  [Frank Harrison]
+- Chore(extra-ctx): sort kwargs passed to jobs. [Frank Harrison]
+- Merge pull request #59 from lursight/chore/cleaner_type_exports.
+  [Frank Harrison]
+
+  Chore/cleaner type exports
+- Chore(clean-types): fixes false-negative from 'isort --check' [Frank
+  Harrison]
+- Chore(clean-types): makes the public_api the actual exported api of
+  the types module. [Frank Harrison]
+
+  I think this could be better
+- Chore(clean-types): renames type_wip -> types. [Frank Harrison]
+- Chore(clean-types): fixes up more of the test-hooks (badly named) for
+  runem itself. [Frank Harrison]
+- Chore(clean-types): splits out the rest of the types. [Frank Harrison]
+- Chore(clean-types): moves config and serialisationtypes to own file.
+  [Frank Harrison]
+- Chore(clean-types): moves error into new errors.py types file. [Frank
+  Harrison]
+- Chore(clean-types): starts moving some types into 'common' [Frank
+  Harrison]
+- Chore(clean-types): moves the types into types/wip whilst I work out
+  the public API. [Frank Harrison]
+- Chore(clean-types): splits out the job and hook execution types, ahead
+  of building a better public api for type exports. [Frank Harrison]
+- Chore(clean-types): sorts the job kwargs. [Frank Harrison]
+
+
+0.1.1 (2024-11-17)
+------------------
+- Release: version 0.1.1 🚀 [Frank Harrison]
+- Merge pull request #58 from lursight/fix/completed_job_counts. [Frank
+  Harrison]
+
+  fix(complete-job-count): fixes and simplifies the in-progress report code
+- Fix(complete-job-count): fixes and simplifies the in-progress report
+  code. [Frank Harrison]
+
+  The core problem was that the remaining-job-count didn't match the
+  number of job-labels that were being shown.
+
+  The root-cause was due to some of the job-tasks completing before the
+  `_update_progress()` thread had started monitoring the jobs.
+
+  We fix this by adding a new parameter to the job-executer that tracks
+  when job completed. It's a very simple fix and reduces the overall
+  complexity for about the same cost of threading-primitives.
+
+
+0.1.0 (2024-11-17)
+------------------
+- Release: version 0.1.0 🚀 [Frank Harrison]
 - Merge pull request #57 from lursight/feat/replace_halo_with_rich.
   [Frank Harrison]
 

{runem-0.1.0 → runem-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.1.0
+Version: 0.1.2
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight

runem-0.1.2/runem/VERSION

@@ -0,0 +1 @@
+0.1.2

{runem-0.1.0 → runem-0.1.2}/runem/cli/initialise_options.py

@@ -2,7 +2,7 @@ import argparse
 
 from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import InformativeDict
-from runem.types import OptionsWritable
+from runem.types.options import OptionsWritable
 
 
 def initialise_options(

{runem-0.1.0 → runem-0.1.2}/runem/command_line.py

@@ -8,7 +8,9 @@ from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import InformativeDict
 from runem.log import error, log
 from runem.runem_version import get_runem_version
-from runem.types import JobNames, OptionConfig, OptionsWritable
+from runem.types.common import JobNames
+from runem.types.options import OptionsWritable
+from runem.types.runem_config import OptionConfig
 from runem.utils import printable_set
 
 

{runem-0.1.0 → runem-0.1.2}/runem/config.py

@@ -7,7 +7,12 @@ from packaging.version import Version
 
 from runem.log import error, log
 from runem.runem_version import get_runem_version
-from runem.types import Config, GlobalConfig, GlobalSerialisedConfig, UserConfigMetadata
+from runem.types.runem_config import (
+    Config,
+    GlobalConfig,
+    GlobalSerialisedConfig,
+    UserConfigMetadata,
+)
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 

{runem-0.1.0 → runem-0.1.2}/runem/config_metadata.py

@@ -3,16 +3,10 @@ import pathlib
 import typing
 
 from runem.informative_dict import InformativeDict
-from runem.types import (
-    JobNames,
-    JobPhases,
-    JobTags,
-    OptionConfigs,
-    OptionsWritable,
-    OrderedPhases,
-    PhaseGroupedJobs,
-    TagFileFilters,
-)
+from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases
+from runem.types.filters import TagFileFilters
+from runem.types.options import OptionsWritable
+from runem.types.runem_config import OptionConfigs, PhaseGroupedJobs
 
 if typing.TYPE_CHECKING:  # pragma: no cover
     from runem.hook_manager import HookManager

{runem-0.1.0 → runem-0.1.2}/runem/config_parse.py

@@ -10,28 +10,23 @@ from runem.hook_manager import HookManager
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log, warn
-from runem.types import (
+from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases, PhaseName
+from runem.types.errors import FunctionNotFound
+from runem.types.filters import TagFileFilter, TagFileFilters
+from runem.types.hooks import HookName
+from runem.types.runem_config import (
     Config,
     ConfigNodes,
-    FunctionNotFound,
     GlobalConfig,
     GlobalSerialisedConfig,
     HookConfig,
-    HookName,
     Hooks,
     HookSerialisedConfig,
     JobConfig,
-    JobNames,
-    JobPhases,
     JobSerialisedConfig,
-    JobTags,
     JobWhen,
     OptionConfigs,
-    OrderedPhases,
     PhaseGroupedJobs,
-    PhaseName,
-    TagFileFilter,
-    TagFileFilters,
     TagFileFilterSerialised,
 )
 

{runem-0.1.0 → runem-0.1.2}/runem/files.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from subprocess import check_output as subprocess_check_output
 
 from runem.config_metadata import ConfigMetadata
-from runem.types import FilePathListLookup
+from runem.types.filters import FilePathListLookup
 
 
 def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:

{runem-0.1.0 → runem-0.1.2}/runem/hook_manager.py

@@ -7,15 +7,10 @@ from runem.config_metadata import ConfigMetadata
 from runem.job import Job
 from runem.job_execute import job_execute
 from runem.log import log
-from runem.types import (
-    FilePathListLookup,
-    HookConfig,
-    HookName,
-    Hooks,
-    HookSpecificKwargs,
-    HooksStore,
-    JobConfig,
-)
+from runem.types.filters import FilePathListLookup
+from runem.types.hooks import HookName
+from runem.types.runem_config import HookConfig, Hooks, HooksStore, JobConfig
+from runem.types.types_jobs import HookSpecificKwargs
 
 
 class HookManager:
@@ -95,6 +90,7 @@ class HookManager:
             job_execute(
                 job_config,
                 running_jobs={},
+                completed_jobs={},
                 config_metadata=config_metadata,
                 file_lists=file_lists,
                 **kwargs,

{runem-0.1.0 → runem-0.1.2}/runem/job.py

@@ -1,6 +1,8 @@
 import typing
 
-from runem.types import FilePathList, FilePathListLookup, JobConfig, JobTags, JobWhen
+from runem.types.common import FilePathList, JobTags
+from runem.types.filters import FilePathListLookup
+from runem.types.runem_config import JobConfig, JobWhen
 
 
 class NoJobName(ValueError):

{runem-0.1.0 → runem-0.1.2}/runem/job_execute.py

@@ -12,14 +12,15 @@ from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log
-from runem.types import (
-    FilePathList,
-    FilePathListLookup,
+from runem.types.common import FilePathList, JobTags
+from runem.types.filters import FilePathListLookup
+from runem.types.runem_config import JobConfig
+from runem.types.types_jobs import (
+    AllKwargs,
     HookSpecificKwargs,
-    JobConfig,
     JobFunction,
+    JobKwargs,
     JobReturn,
-    JobTags,
     JobTiming,
     TimingEntries,
     TimingEntry,
@@ -80,19 +81,27 @@ def job_execute_inner(
         log(f"job: running: '{Job.get_job_name(job_config)}'")
     reports: JobReturn
     try:
-        assert isinstance(function, JobFunction)
-        reports = function(
-            options=ReadOnlyInformativeDict(config_metadata.options),
-            file_list=file_list,
-            procs=config_metadata.args.procs,
-            root_path=root_path,
-            verbose=config_metadata.args.verbose,
-            # unpack useful data points from the job_config
-            label=Job.get_job_name(job_config),
-            job=job_config,
-            record_sub_job_time=_record_sub_job_time,
+        # Define the common args for all jobs and hooks.
+        job_k_args: JobKwargs = {
+            "config_metadata": config_metadata,
+            "file_list": file_list,
+            "job": job_config,
+            "label": Job.get_job_name(job_config),
+            "options": ReadOnlyInformativeDict(config_metadata.options),
+            "procs": config_metadata.args.procs,
+            "record_sub_job_time": _record_sub_job_time,
+            "root_path": root_path,
+            "verbose": config_metadata.args.verbose,
+        }
+        # Merge in the hook-specific kwargs (if any) for situations where we are
+        # calling hooks.
+        all_k_args: AllKwargs = {
+            **job_k_args,
             **kwargs,
-        )
+        }
+
+        assert isinstance(function, JobFunction)
+        reports = function(**all_k_args)
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
         log(decorate=False)
@@ -111,6 +120,7 @@ def job_execute_inner(
 def job_execute(
     job_config: JobConfig,
     running_jobs: typing.Dict[str, str],
+    completed_jobs: typing.Dict[str, str],
     config_metadata: ConfigMetadata,
     file_lists: FilePathListLookup,
     **kwargs: Unpack[HookSpecificKwargs],
@@ -127,5 +137,6 @@ def job_execute(
         file_lists,
         **kwargs,
     )
+    completed_jobs[this_id] = running_jobs[this_id]
     del running_jobs[this_id]
     return results

{runem-0.1.0 → runem-0.1.2}/runem/job_filter.py

@@ -4,14 +4,8 @@ from collections import defaultdict
 from runem.config_metadata import ConfigMetadata
 from runem.job import Job
 from runem.log import log
-from runem.types import (
-    JobConfig,
-    JobNames,
-    JobPhases,
-    JobTags,
-    PhaseGroupedJobs,
-    PhaseName,
-)
+from runem.types.common import JobNames, JobPhases, JobTags, PhaseName
+from runem.types.runem_config import JobConfig, PhaseGroupedJobs
 from runem.utils import printable_set
 
 

{runem-0.1.0 → runem-0.1.2}/runem/job_runner_simple_command.py

@@ -2,7 +2,7 @@ import shlex
 import typing
 
 from runem.run_command import run_command
-from runem.types import JobConfig
+from runem.types.runem_config import JobConfig
 
 
 def validate_simple_command(command_string: str) -> typing.List[str]:

{runem-0.1.0 → runem-0.1.2}/runem/job_wrapper.py

@@ -5,7 +5,8 @@ from runem.job_runner_simple_command import (
     validate_simple_command,
 )
 from runem.job_wrapper_python import get_job_wrapper_py_func
-from runem.types import JobFunction, JobWrapper
+from runem.types.runem_config import JobWrapper
+from runem.types.types_jobs import JobFunction
 
 
 def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobFunction:

{runem-0.1.0 → runem-0.1.2}/runem/job_wrapper_python.py

@@ -3,7 +3,9 @@ import sys
 from importlib.util import module_from_spec
 from importlib.util import spec_from_file_location as module_spec_from_file_location
 
-from runem.types import FunctionNotFound, JobFunction, JobWrapper
+from runem.types.errors import FunctionNotFound
+from runem.types.runem_config import JobWrapper
+from runem.types.types_jobs import JobFunction
 
 
 def _load_python_function_from_module(

{runem-0.1.0 → runem-0.1.2}/runem/report.py

@@ -4,14 +4,13 @@ from collections import defaultdict
 from datetime import timedelta
 
 from runem.log import log
-from runem.types import (
+from runem.types.common import OrderedPhases, PhaseName
+from runem.types.types_jobs import (
     JobReturn,
     JobRunMetadatasByPhase,
     JobRunReportByPhase,
     JobRunTimesByPhase,
     JobTiming,
-    OrderedPhases,
-    PhaseName,
     ReportUrlInfo,
     ReportUrls,
     TimingEntries,

{runem-0.1.0 → runem-0.1.2}/runem/runem.py

@@ -28,7 +28,7 @@ import typing
 from collections import defaultdict
 from datetime import timedelta
 from itertools import repeat
-from multiprocessing.managers import DictProxy, ListProxy, ValueProxy
+from multiprocessing.managers import DictProxy, ValueProxy
 from timeit import default_timer as timer
 from types import TracebackType
 
@@ -41,23 +41,19 @@ from runem.config import load_project_config, load_user_configs
 from runem.config_metadata import ConfigMetadata
 from runem.config_parse import load_config_metadata
 from runem.files import find_files
-from runem.job import Job
 from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
 from runem.log import error, log, warn
 from runem.report import report_on_run
-from runem.types import (
-    Config,
-    FilePathListLookup,
-    HookName,
+from runem.types.common import OrderedPhases, PhaseName
+from runem.types.filters import FilePathListLookup
+from runem.types.hooks import HookName
+from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
+from runem.types.types_jobs import (
     JobReturn,
     JobRunMetadata,
     JobRunMetadatasByPhase,
-    Jobs,
     JobTiming,
-    OrderedPhases,
-    PhaseGroupedJobs,
-    PhaseName,
 )
 from runem.utils import printable_set
 
@@ -123,7 +119,7 @@ class DummySpinner(ConsoleRenderable):  # pragma: no cover
 def _update_progress(
     label: str,
     running_jobs: typing.Dict[str, str],
-    seen_jobs: typing.List[str],
+    completed_jobs: typing.Dict[str, str],
     all_jobs: Jobs,
     is_running: ValueProxy[bool],
     num_workers: int,
@@ -134,7 +130,6 @@ def _update_progress(
     Args:
         label (str): The identifier.
         running_jobs (Dict[str, str]): The currently running jobs.
-        seen_jobs (List[str]): Jobs that the function has previously tracked.
         all_jobs (Jobs): All jobs, encompassing both completed and running jobs.
         is_running (ValueProxy[bool]): Flag indicating if jobs are still running.
         num_workers (int): Indicates the number of workers performing the jobs.
@@ -146,47 +141,25 @@ def _update_progress(
     else:
         spinner = DummySpinner()
 
-    with rich_console.status(spinner):
-
-        # The set of all job labels, and the set of completed jobs
-        all_job_names: typing.Set[str] = {Job.get_job_name(job) for job in all_jobs}
-        completed_jobs: typing.Set[str] = set()
-
-        # This dataset is used to track changes between iterations
-        last_running_jobs_set: typing.Set[str] = set()
+    last_running_jobs_set: typing.Set[str] = set()
 
+    with rich_console.status(spinner):
         while is_running.value:
             running_jobs_set: typing.Set[str] = set(running_jobs.values())
-            seen_jobs = list(running_jobs_set.union(seen_jobs))  # Update the seen jobs
-
-            # Jobs that have disappeared since last check
-            disappeared_jobs: typing.Set[str] = last_running_jobs_set - running_jobs_set
-
-            # Jobs that have not yet completed
-            remaining_jobs: typing.Set[str] = all_job_names - completed_jobs
 
-            # Check if we're closing to completion
-            workers_retiring: bool = len(remaining_jobs) <= num_workers
-
-            if workers_retiring:
-                # Handle edge case: a task may have disappeared whilst process was sleeping
-                all_completed_jobs: typing.Set[str] = all_job_names - remaining_jobs
-                disappeared_jobs.update(all_completed_jobs - running_jobs_set)
-
-            completed_jobs.update(disappeared_jobs)
-
-            # Prepare progress report
+            # Progress report
             progress: str = f"{len(completed_jobs)}/{len(all_jobs)}"
-            running_jobs_list = printable_set(running_jobs_set)
+            running_jobs_list = printable_set(
+                running_jobs_set
+            )  # Reflect current running jobs accurately
+            report: str = f"{label}: {progress}({num_workers}): {running_jobs_list}"
             if show_spinner:
-                spinner.text = (
-                    f"{label}: {progress}({num_workers}): {running_jobs_list}"
-                )
-
-            # Update the tracked dataset for the next iteration
-            last_running_jobs_set = running_jobs_set
+                spinner.text = report
+            else:
+                if last_running_jobs_set != running_jobs_set:
+                    rich_console.log(report)
 
-            # Sleep to decrease frequency of updates and reduce CPU usage
+            # Sleep for reduced CPU usage
             time.sleep(0.1)
 
 
@@ -225,8 +198,8 @@ def _process_jobs(
     subprocess_error: typing.Optional[BaseException] = None
 
     with multiprocessing.Manager() as manager:
-        seen_jobs: ListProxy[str] = manager.list()
         running_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
+        completed_jobs: DictProxy[typing.Any, typing.Any] = manager.dict()
         is_running: ValueProxy[bool] = manager.Value("b", True)
 
         terminal_writer_process = multiprocessing.Process(
@@ -234,7 +207,7 @@ def _process_jobs(
             args=(
                 phase,
                 running_jobs,
-                seen_jobs,
+                completed_jobs,
                 jobs,
                 is_running,
                 num_concurrent_procs,
@@ -251,6 +224,7 @@ def _process_jobs(
                     zip(
                         jobs,
                         repeat(running_jobs),
+                        repeat(completed_jobs),
                         repeat(config_metadata),
                         repeat(file_lists),
                     ),

runem-0.1.2/runem/types/__init__.py

@@ -0,0 +1,12 @@
+from runem.types.common import FilePathList, JobName
+from runem.types.options import Options
+from runem.types.types_jobs import HookKwargs, JobKwargs, JobReturnData
+
+__all__ = [
+    "FilePathList",
+    "HookKwargs",
+    "JobName",
+    "JobReturnData",
+    "Options",
+    "JobKwargs",
+]

runem-0.1.2/runem/types/common.py

@@ -0,0 +1,12 @@
+import typing
+
+# meta-data types
+JobName = str
+JobTag = str
+JobNames = typing.Set[JobName]
+JobPhases = typing.Set[str]
+JobTags = typing.Set[JobTag]
+PhaseName = str
+OrderedPhases = typing.Tuple[PhaseName, ...]
+FilePathSerialise = str
+FilePathList = typing.List[FilePathSerialise]

runem-0.1.2/runem/types/errors.py

@@ -0,0 +1,4 @@
+class FunctionNotFound(ValueError):
+    """Thrown when the test-function cannot be found."""
+
+    pass

runem-0.1.2/runem/types/filters.py

@@ -0,0 +1,12 @@
+import typing
+
+from runem.types.common import FilePathList, JobTag
+
+
+class TagFileFilter(typing.TypedDict):
+    tag: JobTag
+    regex: str
+
+
+TagFileFilters = typing.Dict[JobTag, TagFileFilter]
+FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]

runem-0.1.2/runem/types/hooks.py

@@ -0,0 +1,15 @@
+import enum
+
+
+class HookName(enum.Enum):
+    """List supported hooks.
+
+    TODO:
+    - before all tasks are run, after config is read
+    - BEFORE_ALL = "before-all"
+    - after all tasks are done, before reporting
+    - AFTER_ALL = "after-all"
+    """
+
+    # at exit
+    ON_EXIT = "on-exit"

runem-0.1.2/runem/types/options.py

@@ -0,0 +1,7 @@
+from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
+
+OptionName = str
+OptionValue = bool
+OptionsWritable = InformativeDict[OptionName, OptionValue]
+OptionsReadOnly = ReadOnlyInformativeDict[OptionName, OptionValue]
+Options = OptionsReadOnly