shexec 2.2.0__py3-none-any.whl

shexec/__init__.py

@@ -0,0 +1,52 @@
+"""shexec: a ChRIS DS plugin that runs a shell-type command over every file of a tree.
+
+Key components:
+    - plugin_build, PluginSpec, Plugin: make a ChRIS plugin from a spec; what a
+      downstream plugin calls
+    - ShExec: pfdo_run's engine, built from the plugin's arguments
+    - ShExecOptions: the options
+    - plugin_run: the body of the plugin's main
+    - the records of pfdo_run and pflog, re-exported through shexec.models
+
+Dependencies:
+    - chris_plugin, pfdo_run, pflog
+"""
+
+from __future__ import annotations
+
+__version__: str = "2.2.0"
+
+from shexec.models import (
+    DirNode,
+    FileJob,
+    HostFacts,
+    JobResult,
+    JobSpec,
+    Plugin,
+    PluginSpec,
+    RunResult,
+    ShExecOptions,
+    Span,
+    StageResult,
+)
+from shexec.plugin import plugin_build
+from shexec.shexec import ShExec, options_build, plugin_run
+
+__all__: list[str] = [
+    "DirNode",
+    "FileJob",
+    "HostFacts",
+    "JobResult",
+    "JobSpec",
+    "Plugin",
+    "PluginSpec",
+    "RunResult",
+    "ShExec",
+    "ShExecOptions",
+    "Span",
+    "StageResult",
+    "__version__",
+    "options_build",
+    "plugin_build",
+    "plugin_run",
+]

shexec/__main__.py

@@ -0,0 +1,156 @@
+"""The ``shexec`` command: a ChRIS DS plugin that runs a command over every file of a tree.
+
+This module is shexec's own call of ``plugin_build``: its name, its description for
+ChRIS, no flag defaults, and its words on the manual page. A plugin built on shexec
+has a module of the same shape with its own spec; see ``shexec.plugin`` and the
+README's "Building a plugin on pl-shexec".
+
+Key components:
+    - SPEC: what shexec says about itself
+    - PLUGIN: the built plugin
+    - parser, main: the plugin's flags and the console script's target
+
+Typical usage:
+    shexec --fileFilter 'glob:*.jpg' --exec "convert %{inputWorkingDir}/%{inputWorkingFile}
+        %{outputWorkingDir}/%{inputWorkingFile|rmext}.png" /incoming /outgoing
+
+Dependencies:
+    - shexec.plugin, shexec.models
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Callable
+
+from pftree.manual import Manual
+
+from shexec import __version__
+from shexec.models import Plugin, PluginSpec
+from shexec.plugin import plugin_build
+
+TITLE: str = "Execute shell-type commands across input spaces"
+CATEGORY: str = "Utility"
+MIN_MEMORY: str = "2Gi"
+MIN_CPU: str = "1000m"
+
+BANNER_ART: str = r"""
+       _            _
+      | |          | |
+ _ __ | |______ ___| |__   _____  _____  ___
+| '_ \| |______/ __| '_ \ / _ \ \/ / _ \/ __|
+| |_) | |      \__ \ | | |  __/>  <  __/ (__
+| .__/|_|      |___/_| |_|\___/_/\_\___|\___|
+| |
+|_|
+"""
+
+BANNER_TITLE: str = "Run shell-type commands over an input tree"
+
+BANNER_TEXT: str = """
+                    Run shell-type commands over an input tree
+
+                             -- version {version} --
+
+        A ChRIS DS plugin that runs one command for every selected file of
+        every directory in its input tree, the command written as a template
+        whose %{{...}} tags name the file and directories in hand. The output
+        tree mirrors the input tree; each command's record lands beside its
+        results. The engine is pfdo_run.
+"""
+
+DESCRIPTION: str = """
+    DESCRIPTION
+
+        ``shexec`` is pfdo_run as a ChRIS plugin. ChRIS gives it an input and
+        an output directory; every other flag is pfdo_run's. The input tree is
+        walked, files are selected in each directory by the filters and
+        --analyzeFileIndex, and the --exec template is run once per selected
+        file with its tags resolved against that directory and file, without
+        a shell. Each job's record, the command, its output, its exit status,
+        the host and the timing, is one line of JSON in jobs.jsonl in its
+        output directory unless --noJobLogging is given; --jobFiles adds the
+        four files beside the result; at --verbosity 2 a job's output is also
+        streamed to the console as it runs.
+
+        The plugin's own run is timed: one line of JSON on standard error, when
+        it began, when it ended, the seconds, and whether it succeeded. CUBE
+        keeps that line with the rest of the plugin's output.
+
+        The exit status is 0 when at least one directory's jobs all succeeded,
+        1 when every directory failed or the run could not start, 2 when --exec
+        is missing. The commands available are those of the container image:
+        the coreutils, zip and unzip, bc, perl, tar, ImageMagick's convert,
+        and the DCMTK tools.
+"""
+
+EXAMPLES: str = """
+
+    EXAMPLES
+
+    Convert every jpg in the input tree to png in the output tree:
+
+        shexec                                                                  \\
+            --fileFilter 'glob:*.jpg'                                           \\
+            --exec "convert %{inputWorkingDir}/%{inputWorkingFile}
+                            %{outputWorkingDir}/%{inputWorkingFile|rmext}.png"  \\
+            --threads 4 /incoming /outgoing
+
+    Each conversion's command, stdout, stderr, return code and timing are one
+    line in the directory's jobs.jsonl; ``pflog read /outgoing/**/jobs.jsonl``
+    lists them.
+
+    Copy one file out of each directory whose path names a subject:
+
+        shexec                                                                  \\
+            --dirFilter 100307                                                  \\
+            --exec "cp %{inputWorkingDir}/brain.mgz %{outputWorkingDir}/brain.mgz" \\
+            --analyzeFileIndex f --noJobLogging /incoming /outgoing
+
+    Zip each directory's files into one archive named after the directory:
+
+        shexec                                                                  \\
+            --analyzeFileIndex f                                                \\
+            --exec "zip -j %{outputWorkingDir}/%{inputWorkingDir|basename}.zip
+                           %{inputWorkingDir}/%{inputWorkingFile}"              \\
+            /incoming /outgoing
+
+    From the image, with the directories mounted:
+
+        docker run --rm -v $PWD/in:/incoming:ro -v $PWD/out:/outgoing:rw        \\
+            fnndsc/pl-shexec shexec --fileFilter txt                            \\
+            --exec "wc -l %{inputWorkingDir}/%{inputWorkingFile}" /incoming /outgoing
+"""
+
+
+BRIEF_EXAMPLE: str = """
+
+        shexec --fileFilter 'glob:*.jpg'                                        \\
+            --exec "convert %{inputWorkingDir}/%{inputWorkingFile}
+                            %{outputWorkingDir}/%{inputWorkingFile|rmext}.png"  \\
+            /incoming /outgoing
+"""
+
+SPEC: PluginSpec = PluginSpec(
+    name="shexec",
+    version=__version__,
+    title=TITLE,
+    category=CATEGORY,
+    minMemory=MIN_MEMORY,
+    minCpu=MIN_CPU,
+    bannerArt=BANNER_ART,
+    bannerTitle=BANNER_TITLE,
+    bannerText=BANNER_TEXT,
+    briefExample=BRIEF_EXAMPLE,
+    description=DESCRIPTION,
+    examples=EXAMPLES,
+)
+
+PLUGIN: Plugin = plugin_build(SPEC)
+parser: argparse.ArgumentParser = PLUGIN.parser
+main: Callable[..., int] = PLUGIN.main
+
+
+def manual_build() -> Manual:
+    """The shexec manual page."""
+    return PLUGIN.manual

shexec/models.py

@@ -0,0 +1,152 @@
+"""The records shexec works with: its options, and everything pfdo_run's records carry.
+
+A program built on shexec imports every record from here, and reaches past it into
+pfdo_run, pfdo, pftree, hostlib or pflog for nothing.
+
+Key components:
+    - ShExecOptions: pfdo_run's options, for the plugin
+    - PluginSpec: what a plugin built on shexec says about itself
+    - Plugin: what ``plugin_build`` returns
+    - DirNode, FileJob, JobResult, JobSpec, HostFacts, RunResult, StageResult,
+      ProcessResult, TreeOptions, PfdoOptions, PfdoRunOptions: re-exported
+    - Span: pflog's record of the plugin's run, re-exported
+
+Dependencies:
+    - pfdo_run.models, pflog.models
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from pfdo_run.models import (
+    DirNode,
+    FileJob,
+    HostFacts,
+    JobResult,
+    JobSpec,
+    PfdoOptions,
+    PfdoRunOptions,
+    ProcessResult,
+    RunResult,
+    StageResult,
+    TreeOptions,
+)
+from pflog.models import Span
+
+
+@dataclass
+class ShExecOptions(PfdoRunOptions):
+    """What one run of the plugin is told.
+
+    The same options as pfdo_run, built from the plugin's positional directories
+    and its flags. The class exists so a plugin built on shexec has a place to add
+    its own.
+
+    Example:
+        >>> ShExecOptions.fromDict({"inputDir": "/in", "outputDir": "/out", "exec": "ls"}).exec
+        'ls'
+    """
+
+
+DEFAULT_BANNER_ART: str = r"""
+       _            _
+      | |          | |
+ _ __ | |______ ___| |__   _____  _____  ___
+| '_ \| |______/ __| '_ \ / _ \ \/ / _ \/ __|
+| |_) | |      \__ \ | | |  __/>  <  __/ (__
+| .__/|_|      |___/_| |_|\___/_/\_\___|\___|
+| |
+|_|
+"""
+
+
+@dataclass
+class PluginSpec:
+    """What a plugin built on shexec says about itself.
+
+    Everything a downstream plugin changes from shexec is here; the engine, the
+    parser's flags, the records and the exit status are not, because they are the
+    same for every plugin on the stack.
+
+    Attributes:
+        name: the command and the span, ``shexec``, ``freesurfer``.
+        version: the plugin's version, for ``--version`` and the description.
+        title: the plugin's title in ChRIS.
+        category: the plugin's category in ChRIS.
+        minMemory: the memory ChRIS reserves, as ``2Gi``.
+        minCpu: the CPU ChRIS reserves, in millicores, as ``1000m``.
+        minGpu: GPUs required; 0 for none.
+        defaults: flag defaults, ``exec`` above all; a derived plugin's usual command.
+        bannerArt: the ASCII art on the manual's first page.
+        bannerTitle: the one-line title under it.
+        bannerText: the banner prose; ``{version}`` in it is replaced.
+        briefExample: the manual's BRIEF EXAMPLE, an indented command.
+        description: the manual's DESCRIPTION section, heading included.
+        examples: the manual's EXAMPLES section, heading included.
+
+    Example:
+        >>> PluginSpec(name="freesurfer", version="1.0.0", defaults={"exec": "recon-all"}).defaults["exec"]
+        'recon-all'
+    """
+
+    name: str
+    version: str
+    title: str = ""
+    category: str = "Utility"
+    minMemory: str = "2Gi"
+    minCpu: str = "1000m"
+    minGpu: int = 0
+    defaults: dict[str, Any] = field(default_factory=dict)
+    bannerArt: str = DEFAULT_BANNER_ART
+    bannerTitle: str = ""
+    bannerText: str = ""
+    briefExample: str = ""
+    description: str = ""
+    examples: str = ""
+
+
+@dataclass
+class Plugin:
+    """What ``plugin_build`` returns: a plugin ready to be a console script.
+
+    Attributes:
+        name: the plugin's name.
+        spec: what it was built from.
+        parser: its flags, defaults applied; for ``chris_plugin`` and for tests.
+        main: the decorated main, the console script's target.
+        manual: its manual page.
+
+    Example:
+        >>> Plugin.__dataclass_fields__["main"].name
+        'main'
+    """
+
+    name: str
+    spec: PluginSpec
+    parser: argparse.ArgumentParser
+    main: Callable[..., int]
+    manual: Any
+
+
+__all__: list[str] = [
+    "DEFAULT_BANNER_ART",
+    "DirNode",
+    "FileJob",
+    "HostFacts",
+    "JobResult",
+    "JobSpec",
+    "PfdoOptions",
+    "PfdoRunOptions",
+    "Plugin",
+    "PluginSpec",
+    "ProcessResult",
+    "RunResult",
+    "ShExecOptions",
+    "Span",
+    "StageResult",
+    "TreeOptions",
+]

shexec/plugin.py

@@ -0,0 +1,144 @@
+"""Making a ChRIS plugin out of shexec: the factory a downstream plugin calls.
+
+A plugin that wraps a command-line program is shexec with four things changed: its
+name and version, its description for ChRIS, the defaults of its flags, above all the
+``--exec`` template, and the words on its manual page. ``plugin_build`` takes those as
+a ``PluginSpec`` and returns a ``Plugin``: the parser with the defaults applied, the
+manual composed from the plugin's own banner over pfdo_run's shared sections, and
+``main`` already wrapped in pflog's ``timed`` and ``chris_plugin``. pl-shexec itself is
+the first call of it; pl-freesurfer is the second, with FreeSurfer in the image and
+``recon-all`` as the default template, and no engine code of its own.
+
+Key components:
+    - plugin_build: the factory
+    - synopsis_build: the manual text for a spec
+    - PLUGIN_SIGNATURE: the signature chris_plugin inspects
+
+Typical usage:
+    PLUGIN: Plugin = plugin_build(PluginSpec(
+        name="freesurfer", version=__version__, title="FreeSurfer recon-all",
+        defaults={"exec": RECON_ALL, "analyzeFileIndex": "f", "fileFilter": "glob:*.nii.gz"},
+        minMemory="8Gi", minCpu="4000m",
+    ))
+    main = PLUGIN.main
+
+Dependencies:
+    - chris_plugin, pflog, pfdo_run.__main__, pftree.manual, shexec.shexec, shexec.models
+"""
+
+from __future__ import annotations
+
+import argparse
+import inspect
+from collections.abc import Callable
+from pathlib import Path
+
+from chris_plugin import chris_plugin
+from pfdo_run.__main__ import ARGS_SYNOPSIS_DS, CLI_DS, TAG_HELP, parser_build
+from pflog import timed
+from pftree.manual import Manual, manual_print
+
+from shexec.models import Plugin, PluginSpec
+from shexec.shexec import plugin_run
+
+PLUGIN_SIGNATURE: inspect.Signature = inspect.Signature(
+    [
+        inspect.Parameter("options", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=argparse.Namespace),
+        inspect.Parameter("inputdir", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Path),
+        inspect.Parameter("outputdir", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Path),
+    ],
+    return_annotation=int,
+)
+
+
+def synopsis_build(spec: PluginSpec, shortOnly: bool = False) -> str:
+    """The manual text for a plugin: its NAME, SYNOPSIS and brief example, then the shared sections.
+
+    Args:
+        spec: the plugin.
+        shortOnly: NAME, SYNOPSIS and the brief example only.
+
+    Returns:
+        The text, without the banner.
+    """
+    shortSynopsis: str = (
+        f"""
+    NAME
+
+        {spec.name}
+
+    SYNOPSIS
+
+        {spec.name} \\"""
+        + CLI_DS
+        + "\n\n    BRIEF EXAMPLE\n"
+        + spec.briefExample
+    )
+    if shortOnly:
+        return shortSynopsis
+    return shortSynopsis + spec.description + "\n    ARGS " + ARGS_SYNOPSIS_DS + TAG_HELP + spec.examples
+
+
+def manual_build(spec: PluginSpec) -> Manual:
+    """The manual page for a plugin."""
+    return Manual(
+        command=spec.name,
+        art=spec.bannerArt,
+        title=spec.bannerTitle,
+        text=spec.bannerText,
+        version=spec.version,
+        synopsisShort=synopsis_build(spec, shortOnly=True),
+        synopsisFull=synopsis_build(spec, shortOnly=False),
+    )
+
+
+def parser_forSpec(spec: PluginSpec) -> argparse.ArgumentParser:
+    """pfdo_run's parser named for the plugin, its defaults set from the spec.
+
+    The defaults are set on the parser rather than applied afterwards, so the
+    description ``chris_plugin_info`` serialises carries them and ChRIS shows them.
+    """
+    parser: argparse.ArgumentParser = parser_build(standalone=False, command=spec.name, version=spec.version)
+    parser.set_defaults(**spec.defaults)
+    return parser
+
+
+def plugin_build(spec: PluginSpec, singleton: bool = True) -> Plugin:
+    """Make a ChRIS plugin from a spec.
+
+    Args:
+        spec: name, version, description for ChRIS, flag defaults, manual text.
+        singleton: register with chris_plugin's registry, which ``chris_plugin_info``
+            reads; a test building several plugins in one process passes False.
+
+    Returns:
+        The plugin: its parser, its ``main`` for the console script, and its manual.
+
+    Example:
+        >>> plugin = plugin_build(PluginSpec(name="x", version="0.1"), singleton=False)
+        >>> plugin.name, plugin.parser.get_default("exec")
+        ('x', '')
+    """
+    parser: argparse.ArgumentParser = parser_forSpec(spec)
+    manual: Manual = manual_build(spec)
+
+    def body(options: argparse.Namespace, inputdir: Path, outputdir: Path) -> int:
+        if options.man or options.synopsis:
+            manual_print(manual, shortOnly=not options.man)
+            return 0
+        return plugin_run(options, inputdir, outputdir)
+
+    body.__name__ = spec.name
+    body.__qualname__ = spec.name
+    body.__signature__ = PLUGIN_SIGNATURE  # type: ignore[attr-defined]  # what chris_plugin inspects; this module stringifies annotations
+    timedBody: Callable[[argparse.Namespace, Path, Path], int] = timed(spec.name)(body)
+    main: Callable[..., int] = chris_plugin(
+        parser=parser,
+        title=spec.title,
+        category=spec.category,
+        min_memory_limit=spec.minMemory,
+        min_cpu_limit=spec.minCpu,
+        min_gpu_limit=spec.minGpu,
+        singleton=singleton,
+    )(timedBody)
+    return Plugin(name=spec.name, spec=spec, parser=parser, main=main, manual=manual)

shexec/shexec.py

@@ -0,0 +1,105 @@
+"""What the plugin does: pfdo_run over the directories ChRIS hands it.
+
+A ChRIS DS plugin receives its input and output directories as positional arguments
+and everything else as flags. ``ShExec`` is pfdo_run's engine with its options built
+from those three things, and ``plugin_run`` is the body of the plugin's ``main``:
+configure logging, run, report, and return an exit status.
+
+Key components:
+    - ShExec: pfdo_run's engine, built from the plugin's arguments
+    - options_build: the plugin's arguments as ShExecOptions
+    - plugin_run: run and report, returning the exit status
+
+Typical usage:
+    return plugin_run(options, inputdir, outputdir)
+
+Dependencies:
+    - pfdo_run, pftree.__main__ (logging), shexec.models
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from pfdo_run.pfdo_run import PfdoRun
+from pftree.__main__ import logging_configure
+
+from shexec.models import RunResult, ShExecOptions
+
+EXIT_OK: int = 0
+EXIT_FAILED: int = 1
+EXIT_USAGE: int = 2
+
+
+def options_build(options: argparse.Namespace, inputdir: Path, outputdir: Path) -> ShExecOptions:
+    """The plugin's arguments as options.
+
+    Args:
+        options: the parsed flags.
+        inputdir: the positional input directory.
+        outputdir: the positional output directory.
+
+    Returns:
+        pfdo_run's options with the two directories set.
+
+    Raises:
+        ValueError: a flag's value cannot be coerced to its option's type.
+    """
+    values: dict[str, object] = dict(vars(options))
+    values["inputDir"] = inputdir
+    values["outputDir"] = outputdir
+    return ShExecOptions.fromDict(values)
+
+
+class ShExec(PfdoRun):
+    """pfdo_run as a ChRIS plugin runs it.
+
+    Example:
+        >>> ShExec(ShExecOptions(inputDir=Path("."), outputDir=Path("/tmp/o"), exec="true")).argvTemplate
+        ['true']
+    """
+
+    @classmethod
+    def fromPlugin(cls, options: argparse.Namespace, inputdir: Path, outputdir: Path) -> ShExec:
+        """The engine, built from what the plugin was given."""
+        return cls(options_build(options, inputdir, outputdir))
+
+
+def result_report(options: argparse.Namespace, result: RunResult) -> None:
+    """Print what the flags asked for: the record as JSON, the elapsed time."""
+    if options.json:
+        print(json.dumps(result.toDict(), indent=4, sort_keys=True))
+    if options.printElapsedTime:
+        print(f"Elapsed time = {result.runTime:.6f} seconds")
+
+
+def plugin_run(options: argparse.Namespace, inputdir: Path, outputdir: Path) -> int:
+    """The body of the plugin's main.
+
+    Args:
+        options: the parsed flags; ``exec`` must be set.
+        inputdir: the input tree.
+        outputdir: the output tree.
+
+    Returns:
+        0 when at least one directory's jobs all succeeded (pftree's stage rule);
+        1 when the run failed everywhere or could not start; 2 when ``--exec`` is
+        empty or a flag's value is unusable.
+    """
+    if not options.exec:
+        print("shexec: --exec is required; see shexec --man", file=sys.stderr)
+        return EXIT_USAGE
+    try:
+        engine: ShExec = ShExec.fromPlugin(options, inputdir, outputdir)
+    except ValueError as err:
+        print(f"shexec: {err}", file=sys.stderr)
+        return EXIT_USAGE
+    logging_configure(engine.options.verbosity)
+    result: RunResult = engine.run()
+    result_report(options, result)
+    if result.error:
+        print(result.error, file=sys.stderr)
+    return EXIT_OK if result.status else EXIT_FAILED

shexec-2.2.0.dist-info/METADATA

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: shexec
+Version: 2.2.0
+Summary: Run a shell-type command over every file of an input tree: a ChRIS DS plugin on pfdo_run
+Home-page: https://github.com/FNNDSC/pl-shexec
+Author: FNNDSC
+Author-email: dev@babyMRI.org
+License: MIT
+Project-URL: Homepage, https://github.com/FNNDSC/pl-shexec
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: chris_plugin>=0.4.0
+Requires-Dist: pfdo>=6.2.0
+Requires-Dist: pfdo_run>=4.2.0
+Requires-Dist: hostlib>=1.2.0
+Requires-Dist: pflog>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Dynamic: license-file
+
+pl-shexec
+=========
+
+**Run a command over every file of a tree, as a ChRIS plugin; and the pattern for turning any command-line program into one.**
+
+``pl-shexec`` is a `ChRIS <https://chrisproject.org/>`_ DS plugin that does two things. In a pipeline it is the plugin for the jobs too small to deserve one of their own: convert every image, zip every directory, rename, copy one file out of each subject, count lines. It takes a command template, runs it once per selected file of every directory in its input tree, and writes the results, and a record of each run, into an output tree that mirrors the input.
+
+.. code-block:: bash
+
+    shexec --fileFilter 'glob:*.jpg'                                            \
+        --exec "convert %{inputWorkingDir}/%{inputWorkingFile}
+                        %{outputWorkingDir}/%{inputWorkingFile|rmext}.png"      \
+        /incoming /outgoing
+
+It is also the design for wrapping a large existing program as a plugin. Put the program in the image, set the default command, and the plugin is done: the program's whole command line stays available through the template rather than a hand-picked subset of flags, and the tree walk, the records, the parallelism and the exit status come from the stack. With for example FreeSurfer in the image, one ``recon-all`` per subject directory is:
+
+.. code-block:: bash
+
+    shexec --analyzeFileIndex f --fileFilter 'glob:*.nii.gz'                     \
+        --exec "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile}
+                          -s %{inputWorkingDir|basename} -sd %{outputWorkingDir}" \
+        /incoming /outgoing
+
+pl-freesurfer is an example of the pattern: a plugin that depends on this one the way this one depends on pfdo_run, contributing the FreeSurfer image, that template as the default ``--exec``, and nothing else. See `Building a plugin on pl-shexec`_.
+
+The engine is `pfdo_run <https://github.com/FNNDSC/pfdo_run>`_; the plugin is pfdo_run with its directories supplied by ChRIS and its own run timed.
+
+The template
+------------
+
+``--exec`` is not a shell command. It is a template whose ``%{...}`` tags name the file and directories in hand, resolved once per file, and run directly without a shell, so a path with a space in it stays one argument and nothing is interpreted that was not written. Three tags describe the job:
+
+==========================  ==============================================
+``%{inputWorkingDir}``      the input directory being processed
+``%{outputWorkingDir}``     its counterpart in the output tree
+``%{inputWorkingFile}``     the file being processed
+==========================  ==============================================
+
+Functions derive one name from another: ``%{inputWorkingFile|rmext}.png`` turns ``scan.jpg`` into ``scan.png``; ``%{inputWorkingDir|basename}`` is the directory's own name. Every flag of the plugin is a tag too, and so are ``%{timestamp}``, ``%{hostname}`` and ``%{user}``. The grammar is `pftag <https://github.com/FNNDSC/pftag>`_'s; ``shexec --man`` lists it under TAGS. A template with no tag draws a warning, since the same command would then run for every file.
+
+What each run leaves behind
+---------------------------
+
+Each directory of the output tree gets, beside its results, one file, ``jobs.jsonl``, with one line per job that ran there:
+
+.. code-block:: text
+
+    scan.png
+    jobs.jsonl
+
+The line is the whole record of the job as JSON: the command as executed, everything it printed and everything it complained, its exit status, the host it ran on, and when it started and ended. ``pflog read out/**/jobs.jsonl`` prints a day's jobs as a table, ``--failed`` keeps the ones that went wrong, and a log collector indexes the lines by field.
+
+.. code-block:: text
+
+    STARTED                     DURATION  STATUS  NAME
+    2026-10-14T16:02:07-04:00      4.213  ok      convert /incoming/a/scan.jpg /outgoing/a/scan.png
+    2026-10-14T16:02:12-04:00      0.004  failed  convert /incoming/b/bad.jpg /outgoing/b/bad.png
+
+``--jobFiles`` also writes each job as four files beside its result, ``scan.jpg-cmd``, ``-stdout``, ``-stderr`` and ``-returncode``, the stream files rendered as a terminal would have shown them, for the reader who opens them in an editor. ``--noJobLogging`` keeps no record at all. ``--json`` prints the whole run's record on standard output.
+
+The plugin's own run is timed by `pflog <https://github.com/FNNDSC/pflog>`_: one line of JSON on standard error, when it began, when it ended, the seconds, and whether it succeeded. CUBE keeps that line with the rest of the plugin's output, and any log collector indexes it.
+
+Selecting files and directories
+-------------------------------
+
+The filters are pftree's. ``--fileFilter`` and ``--dirFilter`` take substrings, or ``glob:`` and ``re:`` patterns, joined by ``--fileFilterLogic`` and ``--dirFilterLogic``; ``--maxDepth`` bounds the descent. ``--analyzeFileIndex`` chooses which of a directory's selected files get the command: ``a`` for all, ``f`` for the first, ``l`` for the last, ``m`` for the middle, or an index. ``f`` is how one command runs per directory, started on the first file, as for a DICOM series.
+
+Directories run in parallel under ``--threads``. At ``--verbosity 2`` each command's output streams to the console as it runs.
+
+Exit status
+-----------
+
+0 when at least one directory's jobs all succeeded; 1 when every directory failed or the run could not start; 2 when ``--exec`` is missing or a flag's value is unusable. A failed template, a command that cannot be found, or a non-zero exit fails its directory and the run continues; the record says why.
+
+Wrapping a program
+------------------
+
+A plugin for a program with its own command line, FreeSurfer, ANTs, FSL, DCMTK, is pl-shexec with three changes and no new code:
+
+- the image carries the program, with whatever licence or data it needs;
+- the default ``--exec`` is the program's usual invocation as a template, so the common case needs no command at all and the unusual case overrides it;
+- the default ``--analyzeFileIndex`` chooses the unit, ``f`` for one run per directory, as for a subject or a DICOM series, ``a`` for one run per file.
+
+The user of such a plugin gets the program's entire interface, because anything the program accepts can be written into the template, and gets the same records, the same filters, the same threads and the same truthful exit status as every other plugin on the stack. This is the reason pl-shexec exists in the form it has.
+
+Building a plugin on pl-shexec
+------------------------------
+
+pl-shexec is a library as well as a plugin, in the same way pfdo_run is a library to pl-shexec. A downstream plugin does not copy it; it depends on the ``shexec`` package and calls ``plugin_build`` with a spec of what differs:
+
+.. code-block:: python
+
+    # freesurfer/__main__.py, the whole of pl-freesurfer's code
+    from shexec import PluginSpec, plugin_build
+
+    from freesurfer import __version__
+
+    RECON_ALL: str = (
+        "recon-all -all -i %{inputWorkingDir}/%{inputWorkingFile} "
+        "-s %{inputWorkingDir|basename} -sd %{outputWorkingDir}"
+    )
+
+    PLUGIN = plugin_build(PluginSpec(
+        name="freesurfer",
+        version=__version__,
+        title="FreeSurfer recon-all over every subject",
+        minMemory="8Gi",
+        minCpu="4000m",
+        defaults={"exec": RECON_ALL, "analyzeFileIndex": "f", "fileFilter": "glob:*.nii.gz"},
+        bannerTitle="FreeSurfer recon-all over an input tree",
+        description=DESCRIPTION,
+        examples=EXAMPLES,
+    ))
+    main = PLUGIN.main
+
+``PluginSpec`` holds everything a plugin changes: its name and version, which become its command, its span and its ``--version``; its title, category and resource limits for ChRIS; the defaults of its flags, above all ``--exec``, which ChRIS shows as the defaults and the user can override; and its words on the manual page, over the sections every plugin shares. ``plugin_build`` returns the parser with those defaults applied, the manual, and ``main`` already wrapped in pflog's ``timed`` and ``chris_plugin``, ready to be the console script. The engine, the flags, the records and the exit status are not in the spec, because they are the same for every plugin on the stack, and a fix to them reaches every plugin on its next build.
+
+The rest of pl-freesurfer is a ``pyproject.toml`` naming ``chris_plugin`` and ``shexec`` as its dependencies, the first so that ``chris_plugin_info`` finds the plugin, a ``setup.cfg``, a Dockerfile that starts from the FreeSurfer image and installs the package, tests, and docs. pl-shexec itself is the first plugin built this way; its ``__main__`` is the same call with an empty default command. The pf* stack's ``docs/plugin.adoc`` has the template.
+
+Running it
+----------
+
+From ChRIS, add the plugin to a pipeline and give it ``--exec`` and filters as parameters.
+
+From the command line, with the image:
+
+.. code-block:: bash
+
+    docker run --rm -v $PWD/in:/incoming:ro -v $PWD/out:/outgoing:rw            \
+        fnndsc/pl-shexec shexec --fileFilter txt                                \
+        --exec "wc -l %{inputWorkingDir}/%{inputWorkingFile}" /incoming /outgoing
+
+    docker run --rm fnndsc/pl-shexec shexec --man /tmp /tmp
+
+With Apptainer, ``apptainer exec docker://fnndsc/pl-shexec shexec ...``. The image carries the coreutils, zip and unzip, bc, perl, tar, ImageMagick and the DCMTK tools.
+
+As a Python package, ``pip install .`` in this repository gives the ``shexec`` command and the ``shexec`` module, whose ``plugin_run(options, inputdir, outputdir)`` is the body of the plugin and whose ``shexec.models`` re-exports every record pfdo_run, pfdo, pftree, hostlib and pflog produce.
+
+Development
+-----------
+
+.. code-block:: bash
+
+    uv venv --python /usr/bin/python3.14 .venv
+    uv pip install --python .venv/bin/python -e ".[dev]"
+    .venv/bin/ruff check . && .venv/bin/pytest -q
+    PATH=$PWD/.venv/bin:$PATH chris_plugin_info --dock-image docker.io/fnndsc/pl-shexec:2.1.0 shexec
+
+The CI workflow lints and tests on Python 3.11 to 3.14, builds the image on every push to ``main``, pushes it to Docker Hub and ghcr.io, and on a ``v*`` tag uploads the plugin's description to cube.chrisproject.org and publishes the ``shexec`` package to PyPI, so that a plugin built on pl-shexec can depend on it. The tag must match ``__version__`` in ``shexec/__init__.py``.
+
+Layout
+------
+
+.. code-block:: text
+
+    shexec/
+    ├── __main__.py   the plugin: parser, manual, @chris_plugin @timed main
+    ├── shexec.py     ShExec, options_build, plugin_run
+    └── models.py     ShExecOptions, and every record below re-exported
+
+``docs/purpose.adoc`` says why the plugin exists, ``docs/architecture.adoc`` how it is built, ``docs/history.adoc`` where it came from.

shexec-2.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+shexec/__init__.py,sha256=JBWyItKMRZc6meHZAr8whp1rgtP5uYRoMr-8JOKoZ0M,1142
+shexec/__main__.py,sha256=7PRKuS0p5mEesYwcCdnot5vF_SiTAPKPuMWTt30hsuw,5943
+shexec/models.py,sha256=WXpSzJunootUAOyUDb0MLhmsxiKhHQp3CWNZH272wz4,4361
+shexec/plugin.py,sha256=yveDL-ZfAE8VpRNKqt5gCRylmM_KgVnigqiFu_MVo5M,5353
+shexec/shexec.py,sha256=Uh8Y30Y3lA9S1Tp_UpyG9tGd5KqhKQPShP4_t4DBwSU,3497
+shexec-2.2.0.dist-info/licenses/LICENSE,sha256=X3Pi8IcuV0-qXGRNP4KNpb7OIorb1xqK9dyfaVUJOEc,1069
+shexec-2.2.0.dist-info/METADATA,sha256=EUI8ykdMXYKUHQff7js0NRTgiYQIYu4sYflI_LEFwPs,11822
+shexec-2.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+shexec-2.2.0.dist-info/entry_points.txt,sha256=5IpO-OMKqwNt0icC2XOShqP1hYeHe_wUKQ1weNhP3sk,48
+shexec-2.2.0.dist-info/top_level.txt,sha256=Hx8DpB-sDGRc7bjHLX5XBkL69kOZPLPK3IYtufJ5Px4,7
+shexec-2.2.0.dist-info/RECORD,,

shexec-2.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

shexec-2.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+shexec = shexec.__main__:main

shexec-2.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 FNNDSC / BCH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

shexec-2.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+shexec