runem 0.1.1__tar.gz → 0.2.0__tar.gz

{runem-0.1.1 → runem-0.2.0}/HISTORY.md

@@ -4,6 +4,83 @@ Changelog
 
 (unreleased)
 ------------
+- Merge pull request #63 from
+  lursight/dependabot/github_actions/actions/cache-4. [Frank Harrison]
+
+  chore(deps): bump actions/cache from 3 to 4
+- Chore(deps): bump actions/cache from 3 to 4. [dependabot[bot]]
+
+  Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4.
+  - [Release notes](https://github.com/actions/cache/releases)
+  - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md)
+  - [Commits](https://github.com/actions/cache/compare/v3...v4)
+
+  ---
+  updated-dependencies:
+  - dependency-name: actions/cache
+    dependency-type: direct:production
+    update-type: version-update:semver-major
+  ...
+- Feat(simple-files): adds ability to pass file-lists to simple-
+  commands. [Frank Harrison]
+- Feat(simple-files): strongly types the simple job executor. [Frank
+  Harrison]
+- Feat(simple-files): fixes docstring. [Frank Harrison]
+
+
+0.1.2 (2024-11-18)
+------------------
+- Release: version 0.1.2 🚀 [Frank Harrison]
+- Chore(actions): simplifies code paths for actions. [Frank Harrison]
+- Chore(contrib): removes the Makefile commands that have been removed.
+  [Frank Harrison]
+
+  .. this should have been in the last PR.
+- Chore(contrib): update the CONTRIBUTING docs and removes defunct
+  scripts from Makefile. [Frank Harrison]
+- Chore(faster-actions): removes redundant github actions. [Frank
+  Harrison]
+
+  This should speed up github actions, but it may lead to more false-positives as we are relying on runem to be working and realiabel to run our own checks....
+
+  This could come back to bite us.
+- Chore(extra-ctx): adds config_metadata to job kw-args. [Frank
+  Harrison]
+
+  This allows much greater depth of testing based on other jobs-inclusion and metadata.
+- Chore(extra-ctx): moves the job-task calling to use kwargs explicitly.
+  [Frank Harrison]
+- Chore(extra-ctx): sort kwargs passed to jobs. [Frank Harrison]
+- Merge pull request #59 from lursight/chore/cleaner_type_exports.
+  [Frank Harrison]
+
+  Chore/cleaner type exports
+- Chore(clean-types): fixes false-negative from 'isort --check' [Frank
+  Harrison]
+- Chore(clean-types): makes the public_api the actual exported api of
+  the types module. [Frank Harrison]
+
+  I think this could be better
+- Chore(clean-types): renames type_wip -> types. [Frank Harrison]
+- Chore(clean-types): fixes up more of the test-hooks (badly named) for
+  runem itself. [Frank Harrison]
+- Chore(clean-types): splits out the rest of the types. [Frank Harrison]
+- Chore(clean-types): moves config and serialisationtypes to own file.
+  [Frank Harrison]
+- Chore(clean-types): moves error into new errors.py types file. [Frank
+  Harrison]
+- Chore(clean-types): starts moving some types into 'common' [Frank
+  Harrison]
+- Chore(clean-types): moves the types into types/wip whilst I work out
+  the public API. [Frank Harrison]
+- Chore(clean-types): splits out the job and hook execution types, ahead
+  of building a better public api for type exports. [Frank Harrison]
+- Chore(clean-types): sorts the job kwargs. [Frank Harrison]
+
+
+0.1.1 (2024-11-17)
+------------------
+- Release: version 0.1.1 🚀 [Frank Harrison]
 - Merge pull request #58 from lursight/fix/completed_job_counts. [Frank
   Harrison]
 

{runem-0.1.1 → runem-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: runem
-Version: 0.1.1
+Version: 0.2.0
 Summary: Awesome runem created by lursight
 Home-page: https://github.com/lursight/runem/
 Author: lursight

runem-0.2.0/runem/VERSION

@@ -0,0 +1 @@
+0.2.0

{runem-0.1.1 → runem-0.2.0}/runem/cli/initialise_options.py

@@ -2,7 +2,7 @@ import argparse
 
 from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import InformativeDict
-from runem.types import OptionsWritable
+from runem.types.options import OptionsWritable
 
 
 def initialise_options(

{runem-0.1.1 → runem-0.2.0}/runem/command_line.py

@@ -8,7 +8,9 @@ from runem.config_metadata import ConfigMetadata
 from runem.informative_dict import InformativeDict
 from runem.log import error, log
 from runem.runem_version import get_runem_version
-from runem.types import JobNames, OptionConfig, OptionsWritable
+from runem.types.common import JobNames
+from runem.types.options import OptionsWritable
+from runem.types.runem_config import OptionConfig
 from runem.utils import printable_set
 
 

{runem-0.1.1 → runem-0.2.0}/runem/config.py

@@ -7,7 +7,12 @@ from packaging.version import Version
 
 from runem.log import error, log
 from runem.runem_version import get_runem_version
-from runem.types import Config, GlobalConfig, GlobalSerialisedConfig, UserConfigMetadata
+from runem.types.runem_config import (
+    Config,
+    GlobalConfig,
+    GlobalSerialisedConfig,
+    UserConfigMetadata,
+)
 
 CFG_FILE_YAML = pathlib.Path(".runem.yml")
 

{runem-0.1.1 → runem-0.2.0}/runem/config_metadata.py

@@ -3,16 +3,10 @@ import pathlib
 import typing
 
 from runem.informative_dict import InformativeDict
-from runem.types import (
-    JobNames,
-    JobPhases,
-    JobTags,
-    OptionConfigs,
-    OptionsWritable,
-    OrderedPhases,
-    PhaseGroupedJobs,
-    TagFileFilters,
-)
+from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases
+from runem.types.filters import TagFileFilters
+from runem.types.options import OptionsWritable
+from runem.types.runem_config import OptionConfigs, PhaseGroupedJobs
 
 if typing.TYPE_CHECKING:  # pragma: no cover
     from runem.hook_manager import HookManager

{runem-0.1.1 → runem-0.2.0}/runem/config_parse.py

@@ -10,28 +10,23 @@ from runem.hook_manager import HookManager
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log, warn
-from runem.types import (
+from runem.types.common import JobNames, JobPhases, JobTags, OrderedPhases, PhaseName
+from runem.types.errors import FunctionNotFound
+from runem.types.filters import TagFileFilter, TagFileFilters
+from runem.types.hooks import HookName
+from runem.types.runem_config import (
     Config,
     ConfigNodes,
-    FunctionNotFound,
     GlobalConfig,
     GlobalSerialisedConfig,
     HookConfig,
-    HookName,
     Hooks,
     HookSerialisedConfig,
     JobConfig,
-    JobNames,
-    JobPhases,
     JobSerialisedConfig,
-    JobTags,
     JobWhen,
     OptionConfigs,
-    OrderedPhases,
     PhaseGroupedJobs,
-    PhaseName,
-    TagFileFilter,
-    TagFileFilters,
     TagFileFilterSerialised,
 )
 

{runem-0.1.1 → runem-0.2.0}/runem/files.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from subprocess import check_output as subprocess_check_output
 
 from runem.config_metadata import ConfigMetadata
-from runem.types import FilePathListLookup
+from runem.types.filters import FilePathListLookup
 
 
 def find_files(config_metadata: ConfigMetadata) -> FilePathListLookup:

{runem-0.1.1 → runem-0.2.0}/runem/hook_manager.py

@@ -7,15 +7,10 @@ from runem.config_metadata import ConfigMetadata
 from runem.job import Job
 from runem.job_execute import job_execute
 from runem.log import log
-from runem.types import (
-    FilePathListLookup,
-    HookConfig,
-    HookName,
-    Hooks,
-    HookSpecificKwargs,
-    HooksStore,
-    JobConfig,
-)
+from runem.types.filters import FilePathListLookup
+from runem.types.hooks import HookName
+from runem.types.runem_config import HookConfig, Hooks, HooksStore, JobConfig
+from runem.types.types_jobs import HookSpecificKwargs
 
 
 class HookManager:

{runem-0.1.1 → runem-0.2.0}/runem/job.py

@@ -1,6 +1,8 @@
 import typing
 
-from runem.types import FilePathList, FilePathListLookup, JobConfig, JobTags, JobWhen
+from runem.types.common import FilePathList, JobTags
+from runem.types.filters import FilePathListLookup
+from runem.types.runem_config import JobConfig, JobWhen
 
 
 class NoJobName(ValueError):

{runem-0.1.1 → runem-0.2.0}/runem/job_execute.py

@@ -12,14 +12,15 @@ from runem.informative_dict import ReadOnlyInformativeDict
 from runem.job import Job
 from runem.job_wrapper import get_job_wrapper
 from runem.log import error, log
-from runem.types import (
-    FilePathList,
-    FilePathListLookup,
+from runem.types.common import FilePathList, JobTags
+from runem.types.filters import FilePathListLookup
+from runem.types.runem_config import JobConfig
+from runem.types.types_jobs import (
+    AllKwargs,
     HookSpecificKwargs,
-    JobConfig,
     JobFunction,
+    JobKwargs,
     JobReturn,
-    JobTags,
     JobTiming,
     TimingEntries,
     TimingEntry,
@@ -80,19 +81,27 @@ def job_execute_inner(
         log(f"job: running: '{Job.get_job_name(job_config)}'")
     reports: JobReturn
     try:
-        assert isinstance(function, JobFunction)
-        reports = function(
-            options=ReadOnlyInformativeDict(config_metadata.options),
-            file_list=file_list,
-            procs=config_metadata.args.procs,
-            root_path=root_path,
-            verbose=config_metadata.args.verbose,
-            # unpack useful data points from the job_config
-            label=Job.get_job_name(job_config),
-            job=job_config,
-            record_sub_job_time=_record_sub_job_time,
+        # Define the common args for all jobs and hooks.
+        job_k_args: JobKwargs = {
+            "config_metadata": config_metadata,
+            "file_list": file_list,
+            "job": job_config,
+            "label": Job.get_job_name(job_config),
+            "options": ReadOnlyInformativeDict(config_metadata.options),
+            "procs": config_metadata.args.procs,
+            "record_sub_job_time": _record_sub_job_time,
+            "root_path": root_path,
+            "verbose": config_metadata.args.verbose,
+        }
+        # Merge in the hook-specific kwargs (if any) for situations where we are
+        # calling hooks.
+        all_k_args: AllKwargs = {
+            **job_k_args,
             **kwargs,
-        )
+        }
+
+        assert isinstance(function, JobFunction)
+        reports = function(**all_k_args)
     except BaseException:  # pylint: disable=broad-exception-caught
         # log that we hit an error on this job and re-raise
         log(decorate=False)

{runem-0.1.1 → runem-0.2.0}/runem/job_filter.py

@@ -4,14 +4,8 @@ from collections import defaultdict
 from runem.config_metadata import ConfigMetadata
 from runem.job import Job
 from runem.log import log
-from runem.types import (
-    JobConfig,
-    JobNames,
-    JobPhases,
-    JobTags,
-    PhaseGroupedJobs,
-    PhaseName,
-)
+from runem.types.common import JobNames, JobPhases, JobTags, PhaseName
+from runem.types.runem_config import JobConfig, PhaseGroupedJobs
 from runem.utils import printable_set
 
 

runem-0.2.0/runem/job_runner_simple_command.py

@@ -0,0 +1,46 @@
+import shlex
+import typing
+
+from typing_extensions import Unpack
+
+from runem.run_command import run_command
+from runem.types.common import FilePathList
+from runem.types.runem_config import JobConfig
+from runem.types.types_jobs import AllKwargs
+
+
+def validate_simple_command(command_string: str) -> typing.List[str]:
+    """Use shlex to handle parsing of the command string, a non-trivial problem."""
+    split_command: typing.List[str] = shlex.split(command_string)
+    return split_command
+
+
+def job_runner_simple_command(
+    **kwargs: Unpack[AllKwargs],
+) -> None:
+    """Parses the command and tries to run it via the system.
+
+    Commands inherit the environment.
+    """
+    # assume we have the job.command entry, allowing KeyError to propagate up
+    job_config: JobConfig = kwargs["job"]
+    command_string: str = job_config["command"]
+
+    command_string_files: str = command_string
+    if "{file_list}" in command_string:
+        file_list: FilePathList = kwargs["file_list"]
+        file_list_with_quotes: typing.List[str] = [
+            f'"{str(file_path)}"' for file_path in file_list
+        ]
+        command_string_files = command_string.replace(
+            "{file_list}", " ".join(file_list_with_quotes)
+        )
+
+    # use shlex to handle parsing of the command string, a non-trivial problem.
+    cmd = validate_simple_command(command_string_files)
+
+    # preserve quotes for consistent handling of strings and avoid the "word
+    # splitting" problem for unix-like shells.
+    cmd_with_quotes = [f'"{token}"' if " " in token else token for token in cmd]
+
+    run_command(cmd=cmd_with_quotes, **kwargs)

{runem-0.1.1 → runem-0.2.0}/runem/job_wrapper.py

@@ -5,7 +5,8 @@ from runem.job_runner_simple_command import (
     validate_simple_command,
 )
 from runem.job_wrapper_python import get_job_wrapper_py_func
-from runem.types import JobFunction, JobWrapper
+from runem.types.runem_config import JobWrapper
+from runem.types.types_jobs import JobFunction
 
 
 def get_job_wrapper(job_wrapper: JobWrapper, cfg_filepath: pathlib.Path) -> JobFunction:

{runem-0.1.1 → runem-0.2.0}/runem/job_wrapper_python.py

@@ -3,7 +3,9 @@ import sys
 from importlib.util import module_from_spec
 from importlib.util import spec_from_file_location as module_spec_from_file_location
 
-from runem.types import FunctionNotFound, JobFunction, JobWrapper
+from runem.types.errors import FunctionNotFound
+from runem.types.runem_config import JobWrapper
+from runem.types.types_jobs import JobFunction
 
 
 def _load_python_function_from_module(

{runem-0.1.1 → runem-0.2.0}/runem/report.py

@@ -4,14 +4,13 @@ from collections import defaultdict
 from datetime import timedelta
 
 from runem.log import log
-from runem.types import (
+from runem.types.common import OrderedPhases, PhaseName
+from runem.types.types_jobs import (
     JobReturn,
     JobRunMetadatasByPhase,
     JobRunReportByPhase,
     JobRunTimesByPhase,
     JobTiming,
-    OrderedPhases,
-    PhaseName,
     ReportUrlInfo,
     ReportUrls,
     TimingEntries,

{runem-0.1.1 → runem-0.2.0}/runem/runem.py

@@ -45,18 +45,15 @@ from runem.job_execute import job_execute
 from runem.job_filter import filter_jobs
 from runem.log import error, log, warn
 from runem.report import report_on_run
-from runem.types import (
-    Config,
-    FilePathListLookup,
-    HookName,
+from runem.types.common import OrderedPhases, PhaseName
+from runem.types.filters import FilePathListLookup
+from runem.types.hooks import HookName
+from runem.types.runem_config import Config, Jobs, PhaseGroupedJobs
+from runem.types.types_jobs import (
     JobReturn,
     JobRunMetadata,
     JobRunMetadatasByPhase,
-    Jobs,
     JobTiming,
-    OrderedPhases,
-    PhaseGroupedJobs,
-    PhaseName,
 )
 from runem.utils import printable_set
 

runem-0.2.0/runem/types/__init__.py

@@ -0,0 +1,12 @@
+from runem.types.common import FilePathList, JobName
+from runem.types.options import Options
+from runem.types.types_jobs import HookKwargs, JobKwargs, JobReturnData
+
+__all__ = [
+    "FilePathList",
+    "HookKwargs",
+    "JobName",
+    "JobReturnData",
+    "Options",
+    "JobKwargs",
+]

runem-0.2.0/runem/types/common.py

@@ -0,0 +1,12 @@
+import typing
+
+# meta-data types
+JobName = str
+JobTag = str
+JobNames = typing.Set[JobName]
+JobPhases = typing.Set[str]
+JobTags = typing.Set[JobTag]
+PhaseName = str
+OrderedPhases = typing.Tuple[PhaseName, ...]
+FilePathSerialise = str
+FilePathList = typing.List[FilePathSerialise]

runem-0.2.0/runem/types/errors.py

@@ -0,0 +1,4 @@
+class FunctionNotFound(ValueError):
+    """Thrown when the test-function cannot be found."""
+
+    pass

runem-0.2.0/runem/types/filters.py

@@ -0,0 +1,12 @@
+import typing
+
+from runem.types.common import FilePathList, JobTag
+
+
+class TagFileFilter(typing.TypedDict):
+    tag: JobTag
+    regex: str
+
+
+TagFileFilters = typing.Dict[JobTag, TagFileFilter]
+FilePathListLookup = typing.DefaultDict[JobTag, FilePathList]

runem-0.2.0/runem/types/hooks.py

@@ -0,0 +1,15 @@
+import enum
+
+
+class HookName(enum.Enum):
+    """List supported hooks.
+
+    TODO:
+    - before all tasks are run, after config is read
+    - BEFORE_ALL = "before-all"
+    - after all tasks are done, before reporting
+    - AFTER_ALL = "after-all"
+    """
+
+    # at exit
+    ON_EXIT = "on-exit"

runem-0.2.0/runem/types/options.py

@@ -0,0 +1,7 @@
+from runem.informative_dict import InformativeDict, ReadOnlyInformativeDict
+
+OptionName = str
+OptionValue = bool
+OptionsWritable = InformativeDict[OptionName, OptionValue]
+OptionsReadOnly = ReadOnlyInformativeDict[OptionName, OptionValue]
+Options = OptionsReadOnly

runem-0.2.0/runem/types/runem_config.py

@@ -0,0 +1,164 @@
+import pathlib
+import typing
+
+from runem.types.common import JobName, JobTags, OrderedPhases, PhaseName
+from runem.types.filters import TagFileFilter
+from runem.types.hooks import HookName
+
+
+class OptionConfig(typing.TypedDict, total=False):
+    """Spec for configuring job option overrides."""
+
+    name: str
+    aliases: typing.Optional[typing.List[str]]
+    alias: typing.Optional[str]
+    default: bool
+    type: str
+    desc: typing.Optional[str]
+
+
+OptionConfigs = typing.Tuple[OptionConfig, ...]
+
+
+class OptionConfigSerialised(typing.TypedDict):
+    """Supports better serialisation of options."""
+
+    option: OptionConfig
+
+
+class JobParamConfig(typing.TypedDict):
+    """Configures what parameters are passed to the test-callable.
+
+    FIXME: this isn't actually used at all, yet
+    """
+
+    limitFilesToGroup: bool  # whether to limit file-set for the job
+
+
+class JobAddressConfig(typing.TypedDict):
+    """Configuration which described a callable to call."""
+
+    file: str  # the file-module where 'function' can be found
+    function: str  # the 'function' in module to run
+
+
+class JobContextConfig(typing.TypedDict, total=False):
+    # what parameters the job needs # DEFUNCT
+    params: typing.Optional[JobParamConfig]
+
+    # the path or paths to run the command in. If given a list the job will be
+    # duplicated for each given path.
+    cwd: typing.Optional[typing.Union[str, typing.List[str]]]
+
+
+class JobWhen(typing.TypedDict, total=False):
+    """Configures WHEN to call the callable i.e. priority."""
+
+    tags: JobTags  # the job tags - used for filtering job-types
+    phase: PhaseName  # the phase when the job should be run
+
+
+class JobWrapper(typing.TypedDict, total=False):
+    """A base-type for jobs, hooks, and things that can be invoked."""
+
+    addr: JobAddressConfig  # which callable to call
+    command: str  # a one-liner command to be run
+
+
+class JobConfig(JobWrapper, total=False):
+    """A dict that defines a job to be run.
+
+    It consists of the label, address, context and filter information
+
+    TODO: make a class variant of this
+    """
+
+    label: JobName  # the name of the job
+    ctx: typing.Optional[JobContextConfig]  # how to call the callable
+    when: JobWhen  # when to call the job
+
+
+Jobs = typing.List[JobConfig]
+
+
+class TagFileFilterSerialised(typing.TypedDict):
+    """Supports better serialisation of TagFileFilters."""
+
+    filter: TagFileFilter
+
+
+class GlobalConfig(typing.TypedDict):
+    """The config for the entire test run."""
+
+    # Phases control the order of jobs, jobs earlier in the stack get run earlier
+    # the core ide here is to ensure that certain types of job-dependencies,
+    # such as code-reformatting jobs run before analysis tools, therefore making
+    # any error messages about the code give consistent line numbers e..g if a
+    # re-formatter edits a file the error line will move and the analysis phase
+    # will report the wrong line.
+    phases: OrderedPhases
+
+    # Options control the extra flags that are optionally consumed by job.
+    # Options configured here are used to set command-line-options. All options
+    # and their current state are passed to each job.
+    options: typing.Optional[typing.List[OptionConfigSerialised]]
+
+    # File filters control which files will be passed to jobs for a given tags.
+    # Job will receive the super-set of files for all that job's tags.
+    files: typing.Optional[typing.List[TagFileFilterSerialised]]
+
+    # Which minimal version of runem does this config support?
+    min_version: typing.Optional[str]
+
+
+class GlobalSerialisedConfig(typing.TypedDict):
+    """Intended to make reading a config file easier.
+
+    Unlike JobSerialisedConfig, this type may not actually help readability.
+
+    An intermediary type for serialisation of the global config, the 'global' resides
+    inside a 'global' key and therefore is easier to find and reason about.
+    """
+
+    config: GlobalConfig
+
+
+class HookConfig(JobWrapper, total=False):
+    """Specification for hooks.
+
+    Like JobConfig with use addr or command to specify what to execute.
+    """
+
+    hook_name: HookName  # the hook for when this is called
+
+
+class HookSerialisedConfig(typing.TypedDict):
+    """Intended to make reading a config file easier.
+
+    Also, unlike JobSerialisedConfig, this type may not actually help readability.
+    """
+
+    hook: HookConfig
+
+
+class JobSerialisedConfig(typing.TypedDict):
+    """Makes serialised configs easier to read.
+
+    An intermediary typ for serialisation as each 'job' resides inside a 'job' key.
+
+    This makes formatting of YAML config _significantly_ easier to understand.
+    """
+
+    job: JobConfig
+
+
+ConfigNodes = typing.Union[
+    GlobalSerialisedConfig, JobSerialisedConfig, HookSerialisedConfig
+]
+# The config format as it is serialised to/from disk
+Config = typing.List[ConfigNodes]
+UserConfigMetadata = typing.List[typing.Tuple[Config, pathlib.Path]]
+Hooks = typing.DefaultDict[HookName, typing.List[HookConfig]]
+# A dictionary to hold hooks, with hook names as keys
+HooksStore = typing.Dict[HookName, typing.List[HookConfig]]
+PhaseGroupedJobs = typing.DefaultDict[PhaseName, Jobs]