pfdo-run 4.0.0__py3-none-any.whl

pfdo_run/__init__.py

@@ -0,0 +1,39 @@
+"""pfdo_run: run a command in every directory of a tree.
+
+The member of the pfdo family whose per-directory operation is a command the user
+writes as a pftag template. ``%{inputWorkingDir}``, ``%{outputWorkingDir}`` and
+``%{inputWorkingFile}`` name the current directory's paths and file; functions such as
+``%{inputWorkingFile|rmext}`` derive output names.
+
+Key components:
+    - PfdoRun: the engine
+    - PfdoRunOptions, FileJob: options in, job records out
+    - __main__: the ``pfdo_run`` command and the parts pl-shexec composes its own from
+
+Typical usage:
+    pfdo_run --inputDir /data --outputDir /out --fileFilter 'glob:*.dcm' --analyzeFileIndex f \\
+        --exec "mri_convert %{inputWorkingDir}/%{inputWorkingFile} %{outputWorkingDir}/T1.nii.gz"
+
+Dependencies:
+    - pfdo, pftag, hostlib
+"""
+
+from __future__ import annotations
+
+__version__: str = "4.0.0"
+
+from pfdo_run.models import DirNode, FileJob, JobResult, PfdoRunOptions, RunResult, StageResult
+from pfdo_run.pfdo_run import PfdoRun, lookups_forJob, sink_build
+
+__all__ = [
+    "DirNode",
+    "FileJob",
+    "JobResult",
+    "PfdoRun",
+    "PfdoRunOptions",
+    "RunResult",
+    "StageResult",
+    "__version__",
+    "lookups_forJob",
+    "sink_build",
+]

pfdo_run/__main__.py

@@ -0,0 +1,338 @@
+"""The ``pfdo_run`` command line and its manual.
+
+The parser and the manual are assembled from pftree's and pfdo's parts plus
+pfdo_run's own, and are exposed in parts again for pl-shexec.
+
+Key components:
+    - CLI_SELF, ARGS_SYNOPSIS_SELF, TAG_HELP, EXAMPLES: pfdo_run's manual pieces
+    - CLI_FULL, CLI_DS, ARGS_SYNOPSIS_FULL, ARGS_SYNOPSIS_DS: the assembled pieces
+    - synopsis, manual_build: the manual
+    - parserSelf_build / parser_build: the parsers
+    - main: the entry point
+
+Typical usage:
+    pfdo_run --inputDir /in --outputDir /out --fileFilter jpg \\
+        --exec "convert %{inputWorkingDir}/%{inputWorkingFile} %{outputWorkingDir}/out.png"
+    pfdo_run --man
+
+Dependencies:
+    - pftree, pfdo (their CLI parts and the manual renderer)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+
+from pfdo.__main__ import ARGS_SYNOPSIS_SELF as PFDO_ARGS_SYNOPSIS_SELF
+from pfdo.__main__ import CLI_SELF as PFDO_CLI_SELF
+from pfdo.__main__ import parserSelf_build as pfdoParserSelf_build
+from pftree.__main__ import (
+    ARGS_SYNOPSIS_CORE,
+    ARGS_SYNOPSIS_DS,
+    ARGS_SYNOPSIS_IO,
+    CLI_CORE,
+    CLI_DS_IO,
+    CLI_IO_CORE,
+    logging_configure,
+    parserCore_build,
+    parserIO_build,
+)
+from pftree.manual import Manual, manual_print
+
+from pfdo_run import __version__
+from pfdo_run.models import PfdoRunOptions, RunResult
+from pfdo_run.pfdo_run import PfdoRun
+
+BANNER_ART: str = r"""
+
+                          __    _
+                         / _|  | |
+                   _ __ | |_ __| | ___   _ __ _   _ _ __
+                  | '_ \|  _/ _` |/ _ \ | '__| | | | '_ \
+                  | |_) | || (_| | (_) || |  | |_| | | | |
+                  | .__/|_| \__,_|\___/ |_|   \__,_|_| |_|
+                  | |               ______
+                  |_|              |______|
+
+"""
+
+BANNER_TITLE: str = "Path-File Do Run"
+
+BANNER_TEXT: str = """
+                            Path-File Do Run
+
+        Recursively walk down a directory tree and run a CLI spec'd app
+        on files in each directory (optionally filtered by some simple
+        expression). Results of each operation are saved in an output tree
+        that preserves the input directory structure.
+
+                             -- version {version} --
+
+        'pfdo_run' demonstrates how to use pftree to traverse
+        directory trees and execute some user specified CLI operation
+        at each directory level (that optionally contains files of interest).
+
+        As part of the "pf*" suite of applications, it is geared to IO as
+        directories. Nested directory trees within some input directory
+        are reconstructed in an output directory, preserving directory
+        structure.
+
+"""
+
+CLI_SELF: str = """
+        --exec <commandTemplate>                                                \\
+        [--noJobLogging]                                                        \\"""
+
+ARGS_SYNOPSIS_SELF: str = """
+        --exec <commandTemplate>
+        The command to run for each selected file of each directory in the
+        input tree, written as a pftag template. See the TAGS section. The
+        template is split like a shell command line, each argument has its
+        tags resolved, and the result is run directly, without a shell: a
+        path with a space in it stays one argument, and shell features such
+        as pipes and redirection are not available.
+
+        [--noJobLogging]
+        If specified, suppress the per-job record. Usually each job that is
+        run leaves, in the output directory, four files:
+
+                <inputWorkingFile>-cmd
+                <inputWorkingFile>-stdout
+                <inputWorkingFile>-stderr
+                <inputWorkingFile>-returncode
+
+        The stream files are written as the command runs and, when it ends,
+        rendered as a terminal would have shown them: a progress bar redrawn
+        a hundred times is one line. By specifying this option, the above
+        files are not recorded.
+
+        At --verbosity 2 and above each command's output is also streamed to
+        the console while it runs, so a long job can be watched. Under
+        --threads the streams of concurrent jobs interleave.
+"""
+
+TAG_HELP: str = """
+    TAGS
+
+        The --exec template is resolved by pftag (see pftag --man or the
+        pftag docs/tags.adoc). A tag is %{name}, optionally piped through
+        functions: %{name|function|function(arg, arg)}. Three tags describe
+        the job:
+
+            %{inputWorkingDir}    the input directory being processed
+            %{outputWorkingDir}   its counterpart in the output tree
+            %{inputWorkingFile}   the file being processed
+
+        and every option of pfdo_run is a tag by name, so %{outputDir} is
+        the output tree's root. The core tags %{timestamp}, %{date},
+        %{hostname}, %{platform}, %{user} and %{pid} are available too.
+
+        Functions derive one name from another:
+
+            %{inputWorkingFile|rmext}             brain.nii.gz -> brain.nii
+            %{inputWorkingFile|rmext|rmext}.jpg   brain.nii.gz -> brain.jpg
+            %{inputWorkingFile|replace(.,-)}      a.b.c -> a-b-c
+            %{inputWorkingFile|re([0-9]+,N)}      im001.dcm -> imN.dcm
+            %{inputWorkingDir|basename}           the directory's own name
+            %{inputWorkingFile|sha256(8)}         a stable hash of the name
+
+        Nothing outside %{...} is a tag, so a literal % in the command is
+        fine. A tag that does not resolve fails that job before it runs;
+        the job's record says why.
+"""
+
+EXAMPLES: str = """
+
+    EXAMPLES
+
+    Convert every jpg in a tree to png in a parallel tree:
+
+        pfdo_run                                                                \\
+            --inputDir /var/www/html/data --fileFilter 'glob:*.jpg'             \\
+            --outputDir /var/www/html/png                                       \\
+            --exec "convert %{inputWorkingDir}/%{inputWorkingFile}
+                            %{outputWorkingDir}/%{inputWorkingFile|rmext}.png"  \\
+            --threads 4 --printElapsedTime
+
+    Each conversion's command, stdout, stderr and return code are written
+    beside the png as <file>-cmd, <file>-stdout, <file>-stderr and
+    <file>-returncode. Add --verbosity 2 to watch the conversions as they run.
+
+    Run one command per DICOM series, started on the first file:
+
+        pfdo_run                                                                \\
+            --inputDir /data --outputDir /out                                   \\
+            --fileFilter 'glob:*.dcm' --analyzeFileIndex f                      \\
+            --exec "mri_convert %{inputWorkingDir}/%{inputWorkingFile}
+                                %{outputWorkingDir}/%{inputWorkingDir|basename}.nii.gz"
+
+    Copy one file out of each directory whose path names a subject:
+
+        pfdo_run                                                                \\
+            --inputDir $PWD/raw --outputDir $PWD/out                            \\
+            --dirFilter 100307                                                  \\
+            --exec "cp %{inputWorkingDir}/brain.mgz %{outputWorkingDir}/brain.mgz" \\
+            --analyzeFileIndex f --noJobLogging --json
+
+"""
+
+CLI_FULL: str = CLI_IO_CORE + CLI_SELF + PFDO_CLI_SELF + CLI_CORE
+ARGS_SYNOPSIS_FULL: str = ARGS_SYNOPSIS_IO + ARGS_SYNOPSIS_SELF + PFDO_ARGS_SYNOPSIS_SELF + ARGS_SYNOPSIS_CORE
+CLI_DS: str = CLI_DS_IO + CLI_SELF + PFDO_CLI_SELF + CLI_CORE
+ARGS_SYNOPSIS_DS: str = ARGS_SYNOPSIS_DS + ARGS_SYNOPSIS_SELF + PFDO_ARGS_SYNOPSIS_SELF + ARGS_SYNOPSIS_CORE
+
+DESCRIPTION: str = """
+    DESCRIPTION
+
+        ``pfdo_run`` walks the input tree with pftree, selects files in each
+        directory with the pfdo filters and --analyzeFileIndex, and runs the
+        --exec command once per selected file, with the command's tags
+        resolved against that directory and file. The output tree mirrors
+        the input tree; each directory's results and job logs land in its
+        counterpart. Directories run in parallel under --threads.
+"""
+
+
+def synopsis(shortOnly: bool = False, command: str = "pfdo_run") -> str:
+    """Assemble the manual text.
+
+    Args:
+        shortOnly: NAME, SYNOPSIS and a brief example only.
+        command: the command name shown, for a package built on pfdo_run.
+
+    Returns:
+        The manual text, without the banner.
+    """
+    shortSynopsis: str = (
+        f"""
+    NAME
+
+        {command}
+
+    SYNOPSIS
+
+        {command} \\"""
+        + CLI_FULL
+        + f"""
+
+    BRIEF EXAMPLE
+
+        {command}                                                                \\
+            --inputDir /var/www/html/data --fileFilter jpg                      \\
+            --outputDir /var/www/html/png                                       \\
+            --exec "convert %{{inputWorkingDir}}/%{{inputWorkingFile}}
+                            %{{outputWorkingDir}}/%{{inputWorkingFile|rmext}}.png"  \\
+            --threads 4 --printElapsedTime
+    """
+    )
+    if shortOnly:
+        return shortSynopsis
+    return shortSynopsis + DESCRIPTION + "\n    ARGS " + ARGS_SYNOPSIS_FULL + TAG_HELP + EXAMPLES
+
+
+def manual_build() -> Manual:
+    """The pfdo_run manual page."""
+    return Manual(
+        command="pfdo_run",
+        art=BANNER_ART,
+        title=BANNER_TITLE,
+        text=BANNER_TEXT,
+        version=__version__,
+        synopsisShort=synopsis(shortOnly=True),
+        synopsisFull=synopsis(shortOnly=False),
+    )
+
+
+def parserSelf_build() -> argparse.ArgumentParser:
+    """pfdo_run's own options.
+
+    Returns:
+        A parent parser with ``--exec`` and ``--noJobLogging``.
+    """
+    parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument(
+        "--exec", dest="exec", default="", help="the command template to run per file (required)"
+    )
+    parser.add_argument(
+        "--noJobLogging",
+        dest="noJobLogging",
+        action="store_true",
+        help="do not write per-job cmd/stdout/stderr/rc",
+    )
+    return parser
+
+
+def parser_build(standalone: bool = True) -> argparse.ArgumentParser:
+    """Assemble the full parser.
+
+    Args:
+        standalone: include ``--inputDir`` / ``--outputDir``. A ChRIS DS plugin
+            passes False and adds its own positional directories.
+
+    Returns:
+        The parser.
+    """
+    parents: list[argparse.ArgumentParser] = [
+        parserCore_build("pfdo_run", __version__),
+        pfdoParserSelf_build(),
+        parserSelf_build(),
+    ]
+    if standalone:
+        parents.insert(0, parserIO_build())
+    return argparse.ArgumentParser(
+        prog="pfdo_run",
+        description="Run a command in every directory of a tree. Full manual: pfdo_run --man",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        parents=parents,
+    )
+
+
+def options_fromArgs(args: argparse.Namespace) -> PfdoRunOptions:
+    """Turn parsed arguments into options."""
+    return PfdoRunOptions.fromDict(vars(args))
+
+
+def result_report(args: argparse.Namespace, result: RunResult) -> None:
+    """Print what the arguments asked for."""
+    if args.json:
+        print(json.dumps(result.toDict(), indent=4, sort_keys=True))
+    if args.printElapsedTime:
+        print(f"Elapsed time = {result.runTime:.6f} seconds")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the command.
+
+    Args:
+        argv: arguments without the program name; ``None`` reads ``sys.argv``.
+
+    Returns:
+        0 when at least one directory's jobs all succeeded (pftree's stage rule);
+        1 when the run failed everywhere or could not start; 2 on bad arguments.
+    """
+    parser: argparse.ArgumentParser = parser_build()
+    args: argparse.Namespace = parser.parse_args(argv)
+    if args.man or args.synopsis:
+        manual_print(manual_build(), shortOnly=not args.man)
+        return 0
+    if not args.inputDir:
+        parser.error("--inputDir is required")
+    if not args.exec:
+        parser.error("--exec is required")
+    try:
+        options: PfdoRunOptions = options_fromArgs(args)
+    except ValueError as err:
+        parser.error(str(err))
+    logging_configure(options.verbosity)
+    result: RunResult = PfdoRun(options).run()
+    result_report(args, result)
+    if result.error:
+        print(result.error, file=sys.stderr)
+    return 0 if result.status else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pfdo_run/models.py

@@ -0,0 +1,92 @@
+"""The data models of pfdo_run.
+
+pfdo_run adds two options to pfdo's: the command template to run in every directory,
+and whether each job's record is written beside its results. A job is one run of the
+command for one file; its record names the file, the command as built, and carries the
+hostlib envelope of the run when the command ran.
+
+pfdo's and hostlib's models are re-exported here so that a tool built on pfdo_run
+imports from pfdo_run alone.
+
+Key components:
+    - PfdoRunOptions: ``PfdoOptions`` plus ``exec`` and ``noJobLogging``
+    - FileJob: one file's job: the command built for it and, if it ran, the run's envelope
+    - DirNode, StageResult, ProcessResult, RunResult, PfdoOptions, TreeOptions: pfdo's models
+    - JobResult, JobSpec, HostFacts: hostlib's models
+
+Typical usage:
+    options = PfdoRunOptions.fromDict(vars(args))
+    PfdoRun(options).run()
+
+Dependencies:
+    - pfdo.models, hostlib.models
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from hostlib.models import HostFacts, JobResult, JobSpec
+from pfdo.models import DirNode, PfdoOptions, ProcessResult, RunResult, StageResult, TreeOptions
+
+
+@dataclass
+class PfdoRunOptions(PfdoOptions):
+    """pfdo's options plus pfdo_run's own.
+
+    Attributes:
+        exec: the command template, a pftag template split like a shell command.
+            Its tags name the current directory's paths and file:
+            ``%{inputWorkingDir}``, ``%{outputWorkingDir}``, ``%{inputWorkingFile}``,
+            plus every option by name, ``%{outputDir}``.
+        noJobLogging: do not write each job's command, stdout, stderr and return code
+            beside its results.
+
+    Example:
+        >>> PfdoRunOptions.fromDict({"inputDir": "/in", "exec": "ls %{inputWorkingDir}"}).exec
+        'ls %{inputWorkingDir}'
+    """
+
+    exec: str = ""
+    noJobLogging: bool = False
+
+
+@dataclass
+class FileJob:
+    """One file's job: the command built for it and, when it ran, the run.
+
+    Attributes:
+        file: the input file the job was built for.
+        argv: the command as built, after tag substitution; the template's own
+            arguments when it could not be resolved; empty when there was no template.
+        status: the command was built, started and exited 0.
+        errors: why not: a tag that did not resolve, a program that could not start,
+            a non-zero exit status.
+        run: hostlib's envelope of the run, with both streams, exit status, host facts
+            and timing; ``None`` when the command was never run.
+
+    Example:
+        >>> FileJob("a.txt", ["true"], True).run is None
+        True
+    """
+
+    file: str
+    argv: list[str]
+    status: bool
+    errors: list[str] = field(default_factory=list)
+    run: JobResult | None = None
+
+
+__all__: list[str] = [
+    "DirNode",
+    "FileJob",
+    "HostFacts",
+    "JobResult",
+    "JobSpec",
+    "PfdoOptions",
+    "PfdoRunOptions",
+    "ProcessResult",
+    "RunResult",
+    "StageResult",
+    "TreeOptions",
+]

pfdo_run/pfdo_run.py

@@ -0,0 +1,168 @@
+"""pfdo_run: run a command in every directory of a tree.
+
+``PfdoRun`` is the member of the pfdo family whose per-directory operation is a command
+written by the user as a pftag template. For every file the directory's
+``--analyzeFileIndex`` selects, the template is split like a shell command, each
+argument has its tags resolved against that directory's paths and the file, and the
+result is run through hostlib: without a shell, both streams read as they arrive, the
+record kept beside the results and, at ``--verbosity 2`` and above, streamed to the
+console while the command runs.
+
+Key components:
+    - PfdoRun: the engine; ``job_run`` builds and runs one command
+    - lookups_forJob: the tags a job can use
+    - sink_build: where a job's output goes while it runs
+
+Typical usage:
+    PfdoRun(PfdoRunOptions(inputDir=Path("/in"), outputDir=Path("/out"),
+            exec="convert %{inputWorkingDir}/%{inputWorkingFile} "
+                 "%{outputWorkingDir}/%{inputWorkingFile|rmext}.jpg")).run()
+
+Dependencies:
+    - pfdo, pftag, hostlib
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import logging
+import shlex
+
+from hostlib.proc.run import job_run as host_jobRun
+from hostlib.sink import ConsoleSink, FileSink, JobSink, NullSink, TeeSink
+from pfdo.pfdo import Pfdo
+from pftag.models import ArgvResult, TagOptions
+from pftag.pftag import Pftag
+
+from pfdo_run.models import DirNode, FileJob, JobResult, JobSpec, PfdoRunOptions, StageResult
+
+logger: logging.Logger = logging.getLogger("pfdo_run")
+
+OPTIONS_LOOKUP: str = "options"
+JOB_LOOKUP: str = "pfdo"
+STREAM_VERBOSITY: int = 2
+
+
+def lookups_forJob(options: PfdoRunOptions, node: DirNode, fileName: str) -> dict[str, dict[str, str]]:
+    """The tag dictionaries one job can resolve against.
+
+    Args:
+        options: the run's options; every field becomes a tag in the ``options``
+            dictionary, so ``%{outputDir}`` works.
+        node: the directory being processed.
+        fileName: the file the job is built for.
+
+    Returns:
+        ``options`` and ``pfdo`` (``inputWorkingDir``, ``outputWorkingDir``,
+        ``inputWorkingFile``), the latter also reachable qualified.
+    """
+    optionTags: dict[str, str] = {
+        name: str(value) for name, value in dataclasses.asdict(options).items() if name not in ("exec",)
+    }
+    jobTags: dict[str, str] = {
+        "inputWorkingDir": str(node.path),
+        "outputWorkingDir": str(node.outputPath),
+        "inputWorkingFile": fileName,
+    }
+    return {OPTIONS_LOOKUP: optionTags, JOB_LOOKUP: jobTags}
+
+
+def sink_build(options: PfdoRunOptions, node: DirNode, fileName: str) -> JobSink:
+    """Where one job's output goes while it runs.
+
+    Args:
+        options: ``noJobLogging`` turns the file record off; ``verbosity`` of
+            ``STREAM_VERBOSITY`` or more streams to the console.
+        node: the directory; the record lands in its ``outputPath``.
+        fileName: the file; the record is ``<file>-cmd``, ``-stdout``, ``-stderr``,
+            ``-returncode``, the stream files rendered as a terminal would show them.
+
+    Returns:
+        A file sink, a console sink, both, or nothing.
+    """
+    sinks: list[JobSink] = []
+    if not options.noJobLogging:
+        sinks.append(FileSink(node.outputPath / fileName, rendered=True))
+    if options.verbosity >= STREAM_VERBOSITY:
+        sinks.append(ConsoleSink())
+    if not sinks:
+        return NullSink()
+    return sinks[0] if len(sinks) == 1 else TeeSink(*sinks)
+
+
+class PfdoRun(Pfdo):
+    """Run a command template in every directory of the input tree.
+
+    Attributes:
+        options: the PfdoRunOptions in force.
+        argvTemplate: the ``--exec`` template split like a shell command, once.
+
+    Example:
+        >>> tool = PfdoRun(PfdoRunOptions(inputDir=Path("."), outputDir=Path("/tmp/o"), exec="true"))
+        >>> tool.argvTemplate
+        ['true']
+    """
+
+    def __init__(self, options: PfdoRunOptions) -> None:
+        super().__init__(options)
+        self.options: PfdoRunOptions = options
+        self.argvTemplate: list[str] = shlex.split(options.exec) if options.exec else []
+
+    def inputRead_callback(self, node: DirNode) -> StageResult:
+        """Create the directory's output path and pass its files on.
+
+        Args:
+            node: the directory.
+
+        Returns:
+            Success with the files; failure if there are none.
+        """
+        node.outputPath.mkdir(parents=True, exist_ok=True)
+        return StageResult(status=bool(node.files), files=list(node.files))
+
+    def inputAnalyze_callback(self, node: DirNode, read: StageResult) -> StageResult:
+        """Run the command for every file ``--analyzeFileIndex`` selects.
+
+        Args:
+            node: the directory.
+            read: the read stage's result.
+
+        Returns:
+            The files run, with every FileJob in ``payload["jobs"]``; ``status`` is
+            True only when every job was built, started and exited 0.
+        """
+        jobs: list[FileJob] = [self.job_run(node, fileName) for fileName in self.filesToAnalyze(node)]
+        return StageResult(
+            status=bool(jobs) and all(job.status for job in jobs),
+            files=[job.file for job in jobs],
+            payload={"jobs": [dataclasses.asdict(job) for job in jobs]},
+        )
+
+    def job_run(self, node: DirNode, fileName: str) -> FileJob:
+        """Build and run the command for one file.
+
+        The template's tags are resolved per argument, so a path with a space stays
+        one argument; the command then runs through hostlib, without a shell, its
+        output going to the sink ``sink_build`` chooses while it runs.
+
+        Args:
+            node: the directory.
+            fileName: the file the command is built for.
+
+        Returns:
+            The job's record. A template that fails to resolve is a job that was
+            never run; a program that cannot start or exits non-zero is a failed run.
+        """
+        if not self.argvTemplate:
+            return FileJob(fileName, [], False, ["--exec is empty"])
+        tagger: Pftag = Pftag(TagOptions(lookups=lookups_forJob(self.options, node, fileName), strict=True))
+        resolved: ArgvResult = tagger.argv_process(self.argvTemplate)
+        if not resolved.status:
+            logger.error("%s/%s: %s", node.relPath, fileName, "; ".join(resolved.errors()))
+            return FileJob(fileName, resolved.argv, False, resolved.errors())
+        logger.debug("%s/%s: %s", node.relPath, fileName, shlex.join(resolved.argv))
+        sink: JobSink = sink_build(self.options, node, fileName)
+        run: JobResult = host_jobRun(JobSpec(argv=resolved.argv), sink=sink)
+        if not run.status:
+            logger.warning("%s/%s: %s", node.relPath, fileName, "; ".join(run.errors))
+        return FileJob(fileName, resolved.argv, run.status, list(run.errors), run)

pfdo_run-4.0.0.dist-info/METADATA

@@ -0,0 +1,110 @@
+Metadata-Version: 2.4
+Name: pfdo_run
+Version: 4.0.0
+Summary: Run a command, written as a pftag template, in every directory of a tree
+Author-email: FNNDSC <dev@babyMRI.org>
+License: MIT
+Project-URL: Homepage, https://github.com/FNNDSC/pfdo_run
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: pfdo>=6.1.0
+Requires-Dist: pftag>=3.1.0
+Requires-Dist: hostlib>=1.1.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"
+Dynamic: license-file
+
+pfdo_run
+========
+
+.. image:: https://github.com/FNNDSC/pfdo_run/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/FNNDSC/pfdo_run/actions/workflows/ci.yml
+
+.. image:: https://img.shields.io/pypi/v/pfdo_run.svg
+    :target: https://pypi.org/project/pfdo_run/
+
+.. image:: https://img.shields.io/badge/python-3.11%2B-blue.svg
+    :target: https://pypi.org/project/pfdo_run/
+
+Run a command, written as a pftag template, in every directory of a tree.
+
+.. contents:: Table of Contents
+
+Overview
+--------
+
+``pfdo_run`` walks an input directory with ``pftree``, selects files in each directory with the ``pfdo`` filters and ``--analyzeFileIndex``, and runs the ``--exec`` command once per selected file. The command is a ``pftag`` template: ``%{inputWorkingDir}``, ``%{outputWorkingDir}`` and ``%{inputWorkingFile}`` name the directory being processed, its counterpart in the output tree, and the file; functions derive names, ``%{inputWorkingFile|rmext}.png``. The output tree mirrors the input tree, and each job leaves its command, stdout, stderr and return code beside its results. Directories run in parallel under ``--threads``.
+
+The template is split like a shell command line and each argument has its tags resolved on its own, so a path with a space stays one argument and nothing a file name contains can change the command. The command runs without a shell.
+
+Installation
+------------
+
+.. code:: bash
+
+    pip install pfdo_run
+
+Python 3.11 or newer; depends on ``pfdo`` and ``pftag``.
+
+Command line
+------------
+
+.. code:: bash
+
+    pfdo_run --inputDir /data --outputDir /png --fileFilter 'glob:*.jpg' --threads 4 \
+        --exec "convert %{inputWorkingDir}/%{inputWorkingFile} %{outputWorkingDir}/%{inputWorkingFile|rmext}.png"
+
+    pfdo_run --inputDir /dicom --outputDir /nifti --fileFilter 'glob:*.dcm' --analyzeFileIndex f \
+        --exec "mri_convert %{inputWorkingDir}/%{inputWorkingFile} %{outputWorkingDir}/%{inputWorkingDir|basename}.nii.gz"
+
+    pfdo_run --man
+
+All of ``pftree``'s and ``pfdo``'s options apply. ``pfdo_run`` adds:
+
+``--exec <commandTemplate>``
+    The command to run per selected file. Tags: ``%{inputWorkingDir}``, ``%{outputWorkingDir}``, ``%{inputWorkingFile}``, every option by name (``%{outputDir}``), and pftag's core tags. Functions: ``rmext``, ``ext``, ``basename``, ``dirname``, ``replace``, ``re``, ``upper``, ``lower``, ``strip``, ``pad``, ``slice``, ``md5``, ``sha256``, ``strftime``, ``mask``. A tag that does not resolve fails that job before it runs.
+
+``--noJobLogging``
+    Do not write ``<file>-cmd``, ``<file>-stdout``, ``<file>-stderr`` and ``<file>-returncode`` beside each job's results. The stream files are written live and rendered as a terminal would show them when the job ends.
+
+A job whose command exits non-zero fails its directory; the run continues and the directory's record says why. ``--json`` prints the whole result, each directory's jobs included.
+
+Library
+-------
+
+.. code:: python
+
+    from pathlib import Path
+    from pfdo_run import PfdoRun, PfdoRunOptions
+
+    options = PfdoRunOptions(
+        inputDir=Path("/data"), outputDir=Path("/png"), fileFilter="glob:*.jpg", threads=4,
+        exec="convert %{inputWorkingDir}/%{inputWorkingFile} %{outputWorkingDir}/%{inputWorkingFile|rmext}.png",
+    )
+    result = PfdoRun(options).run()
+    for key, stage in result.process.analyze.items():
+        for job in stage.payload["jobs"]:
+            print(key, job["file"], job["run"]["returncode"] if job["run"] else job["errors"])
+
+``pfdo_run.__main__`` exposes its parser and manual in parts (``parser_build(standalone=False)``, ``synopsis(command=...)``, ``CLI_DS``, ``ARGS_SYNOPSIS_DS``) so that a ChRIS plugin such as ``pl-shexec`` composes its own command around them.
+
+Changes in 4.0
+--------------
+
+- Built on ``pfdo`` 6 and ``pftag`` 3; ``pfmisc``, ``pudb`` and ``Faker`` are no longer dependencies.
+- The template grammar is pftag's: ``%{name}`` and ``%{name|fn(arg)}``. The 3.x forms ``%name`` and ``%_fn_name`` are gone; a bare ``%name`` of a known tag is reported as a 1.x template and fails the job.
+- The command is split before substitution and run without a shell; 3.x substituted into the string and split on whitespace, so a path with a space broke it.
+- The command runs through ``hostlib``: no shell, both streams read concurrently as they arrive, so nothing is dropped and nothing can deadlock; 3.x read stdout line by line, lost the last line, and could deadlock on stderr. At ``--verbosity 2`` and above the output also streams to the console while the job runs.
+- Per-job records are written by default and ``--noJobLogging`` exists; in 3.x the option was documented but undefined, so logs were never written. The stream files are rendered as a terminal would have shown them, so a redrawn progress bar is one line; the record in ``--json`` keeps every frame.
+- Each job's record is ``FileJob``: the file, the command as built, status, errors, and ``run``, hostlib's envelope of the run (both streams, exit status, host facts, timing) when the command ran.
+- A non-zero exit status fails the job and its directory; the run's status is truthful (3.x reported False always).
+- ``main(argv)`` honours ``argv``; ``--version``, ``--man`` and ``--synopsis`` exit 0; packaging is ``pyproject.toml`` only, published from CI on a ``v*`` tag.

pfdo_run-4.0.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+pfdo_run/__init__.py,sha256=QJGTGgOJw-lqucqSOGh-mwkr3NUR7WLXD-pG8yaIiOI,1225
+pfdo_run/__main__.py,sha256=FNDIhZ99gVFi2Ght7EthEH9BcmvRM3SYpBbkMDaS7OY,12470
+pfdo_run/models.py,sha256=MnUfCDqJDQaBc3_X8jAd9bw4Eg3GDUvSzH9AP47i5wE,3000
+pfdo_run/pfdo_run.py,sha256=QfdA6EuJaHyWk4SOvqhEpkpR4eqdBgwQ-F8cZMH_QDk,6843
+pfdo_run-4.0.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pfdo_run-4.0.0.dist-info/METADATA,sha256=xp24R6Ldd4GNt_xG5en69qVNPIovjziWoQdofcIrY-s,6231
+pfdo_run-4.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pfdo_run-4.0.0.dist-info/entry_points.txt,sha256=2tAd_KG6g-xH-_F5XRX56byHGHirdVZrscY9LV-h-Lw,52
+pfdo_run-4.0.0.dist-info/top_level.txt,sha256=sws_cVZI7zNCzMd4FswHSKRE0TR3VIcWL6Rwg-2LuI4,9
+pfdo_run-4.0.0.dist-info/RECORD,,

pfdo_run-4.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pfdo_run-4.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pfdo_run = pfdo_run.__main__:main

pfdo_run-4.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pfdo_run-4.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pfdo_run