hydraflow 0.13.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

hydraflow/cli.py

@@ -3,18 +3,19 @@
 from __future__ import annotations
 
 import shlex
-from typing import Annotated
+from typing import TYPE_CHECKING, Annotated
 
 import typer
-from rich.console import Console
-from typer import Argument, Option
+from typer import Argument, Exit, Option
+
+if TYPE_CHECKING:
+    from hydraflow.executor.conf import Job
 
 app = typer.Typer(add_completion=False)
-console = Console()
 
 
-@app.command(context_settings={"ignore_unknown_options": True})
-def run(
+@app.command("run", context_settings={"ignore_unknown_options": True})
+def _run(
     name: Annotated[str, Argument(help="Job name.", show_default=False)],
     *,
     args: Annotated[
@@ -28,65 +29,68 @@ def run(
 ) -> None:
     """Run a job."""
     from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import iter_batches, iter_calls, iter_runs
 
     args = args or []
     job = get_job(name)
 
-    if job.run:
-        args = [*shlex.split(job.run), *args]
-        it = iter_runs(args, iter_batches(job), dry_run=dry_run)
+    if not dry_run:
+        import mlflow
+
+        mlflow.set_experiment(job.name)
+
+    if job.submit:
+        submit(job, args, dry_run=dry_run)
+
+    elif job.run:
+        run(job, args, dry_run=dry_run)
+
     elif job.call:
-        args = [*shlex.split(job.call), *args]
-        it = iter_calls(args, iter_batches(job), dry_run=dry_run)
+        call(job, args, dry_run=dry_run)
+
     else:
         typer.echo(f"No command found in job: {job.name}.")
-        raise typer.Exit(1)
+        raise Exit(1)
 
-    if not dry_run:
-        import mlflow
 
-        mlflow.set_experiment(job.name)
+def run(job: Job, args: list[str], *, dry_run: bool) -> None:
+    """Run a job."""
+    from hydraflow.executor import aio
+    from hydraflow.executor.job import iter_batches, iter_tasks
 
-    for task in it:  # jobs will be executed here
-        if job.run and dry_run:
-            typer.echo(shlex.join(task.args))
-        elif job.call and dry_run:
-            funcname, *args = task.args
-            arg = ", ".join(f"{arg!r}" for arg in args)
-            typer.echo(f"{funcname}([{arg}])")
+    args = [*shlex.split(job.run), *args]
+    it = iter_tasks(args, iter_batches(job))
 
+    if not dry_run:
+        aio.run(it)
+        raise Exit
 
-@app.command(context_settings={"ignore_unknown_options": True})
-def submit(
-    name: Annotated[str, Argument(help="Job name.", show_default=False)],
-    *,
-    args: Annotated[
-        list[str] | None,
-        Argument(help="Arguments to pass to the job.", show_default=False),
-    ] = None,
-    dry_run: Annotated[
-        bool,
-        Option("--dry-run", help="Perform a dry run."),
-    ] = False,
-) -> None:
-    """Submit a job."""
-    from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import iter_batches, submit
+    for task in it:
+        typer.echo(shlex.join(task.args))
 
-    args = args or []
-    job = get_job(name)
 
-    if not job.run:
-        typer.echo(f"No run found in job: {job.name}.")
-        raise typer.Exit(1)
+def call(job: Job, args: list[str], *, dry_run: bool) -> None:
+    """Call a job."""
+    from hydraflow.executor.job import iter_batches, iter_calls
+
+    args = [*shlex.split(job.call), *args]
+    it = iter_calls(args, iter_batches(job))
 
     if not dry_run:
-        import mlflow
+        for call in it:
+            call.func()
+        raise Exit
 
-        mlflow.set_experiment(job.name)
+    for task in it:
+        funcname, *args = task.args
+        arg = ", ".join(f"{arg!r}" for arg in args)
+        typer.echo(f"{funcname}([{arg}])")
 
-    args = [*shlex.split(job.run), *args]
+
+def submit(job: Job, args: list[str], *, dry_run: bool) -> None:
+    """Submit a job."""
+    from hydraflow.executor.job import iter_batches, submit
+
+    args = [*shlex.split(job.submit), *args]
     result = submit(args, iter_batches(job), dry_run=dry_run)
 
     if dry_run and isinstance(result, tuple):
@@ -124,4 +128,4 @@ def callback(
         import importlib.metadata
 
         typer.echo(f"hydraflow {importlib.metadata.version('hydraflow')}")
-        raise typer.Exit
+        raise Exit

hydraflow/executor/aio.py

@@ -0,0 +1,85 @@
+"""Asynchronous execution."""
+
+from __future__ import annotations
+
+import asyncio
+from asyncio.subprocess import PIPE
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+if TYPE_CHECKING:
+    from asyncio.streams import StreamReader
+    from collections.abc import Callable, Iterable
+
+    from hydraflow.executor.job import Task
+
+
+console = Console(log_time=False, log_path=False)
+
+
+def run(iterable: Iterable[Task]) -> int | None:
+    """Run multiple tasks."""
+    with Progress(
+        SpinnerColumn(),
+        TimeElapsedColumn(),
+        BarColumn(),
+        TimeRemainingColumn(),
+        MofNCompleteColumn(),
+        TaskProgressColumn(),
+        console=console,
+    ) as progress:
+
+        def stdout(output: str) -> None:
+            progress.log(output.rstrip())
+
+        def stderr(output: str) -> None:
+            progress.log(f"[red]{output}".rstrip())
+
+        task_id = progress.add_task("")
+
+        for task in iterable:
+            progress.update(task_id, total=task.total)
+
+            coro = arun(task.args, stdout, stderr)
+            returncode = asyncio.run(coro)
+
+            if returncode:
+                return returncode
+
+            progress.update(task_id, completed=task.index + 1)
+
+        return 0
+
+
+async def arun(
+    args: list[str],
+    stdout: Callable[[str], None],
+    stderr: Callable[[str], None],
+) -> int | None:
+    """Run a command asynchronously."""
+    process = await asyncio.create_subprocess_exec(*args, stdout=PIPE, stderr=PIPE)
+    coros = alog(process.stdout, stdout), alog(process.stderr, stderr)  # type:ignore
+    await asyncio.gather(*coros)
+    await process.communicate()
+
+    return process.returncode
+
+
+async def alog(reader: StreamReader, write: Callable[[str], None]) -> None:
+    """Log a stream of output asynchronously."""
+    while True:
+        if reader.at_eof():
+            break
+
+        if out := await reader.readline():
+            write(out.decode())

hydraflow/executor/conf.py

@@ -15,6 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
+    submit: str = ""
     with_: str = ""
     steps: list[Step] = field(default_factory=list)
 

hydraflow/executor/job.py

@@ -25,7 +25,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from subprocess import CompletedProcess
 from tempfile import NamedTemporaryFile
-from typing import TYPE_CHECKING, overload
+from typing import TYPE_CHECKING
 
 import ulid
 
@@ -86,47 +86,22 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
 @dataclass
 class Task:
-    """An executed task."""
+    """A task to be executed."""
 
     args: list[str]
     total: int
-    completed: int
-
-
-@dataclass
-class Run(Task):
-    """An executed run."""
-
-    result: CompletedProcess
+    index: int
 
 
 @dataclass
 class Call(Task):
-    """An executed call."""
+    """A call to be executed."""
 
-    result: Any
+    func: Callable[[], Any]
 
 
-@overload
-def iter_runs(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Run]: ...
-
-
-@overload
-def iter_runs(
-    args: list[str],
-    iterable: Iterable[list[str]],
-    *,
-    dry_run: bool = False,
-) -> Iterator[Task | Run]: ...
-
-
-def iter_runs(
-    args: list[str],
-    iterable: Iterable[list[str]],
-    *,
-    dry_run: bool = False,
-) -> Iterator[Task | Run]:
-    """Execute multiple runs of a job using shell commands."""
+def iter_tasks(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Task]:
+    """Yield tasks of a job to be executed using a shell command."""
     executable, *args = args
     if executable == "python" and sys.platform == "win32":
         executable = sys.executable
@@ -134,48 +109,21 @@ def iter_runs(
     iterable = list(iterable)
     total = len(iterable)
 
-    for completed, args_ in enumerate(iterable, 1):
-        cmd = [executable, *args, *args_]
-        if dry_run:
-            yield Task(cmd, total, completed)
-        else:
-            result = subprocess.run(cmd, check=False)
-            yield Run(cmd, total, completed, result)
-
-
-@overload
-def iter_calls(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Call]: ...
+    for index, args_ in enumerate(iterable):
+        yield Task([executable, *args, *args_], total, index)
 
 
-@overload
-def iter_calls(
-    args: list[str],
-    iterable: Iterable[list[str]],
-    *,
-    dry_run: bool = False,
-) -> Iterator[Task | Call]: ...
-
-
-def iter_calls(
-    args: list[str],
-    iterable: Iterable[list[str]],
-    *,
-    dry_run: bool = False,
-) -> Iterator[Task | Call]:
-    """Execute multiple calls of a job using Python functions."""
+def iter_calls(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Call]:
+    """Yield calls of a job to be executed using a Python function."""
     funcname, *args = args
     func = get_callable(funcname)
 
     iterable = list(iterable)
     total = len(iterable)
 
-    for completed, args_ in enumerate(iterable, 1):
+    for index, args_ in enumerate(iterable):
         cmd = [funcname, *args, *args_]
-        if dry_run:
-            yield Task(cmd, total, completed)
-        else:
-            result = func([*args, *args_])
-            yield Call(cmd, total, completed, result)
+        yield Call(cmd, total, index, lambda x=cmd[1:]: func(x))
 
 
 def submit(

{hydraflow-0.13.0.dist-info → hydraflow-0.14.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.13.0
+Version: 0.14.1
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -38,10 +38,10 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: hydra-core>=1.3
 Requires-Dist: mlflow>=2.15
-Requires-Dist: omegaconf
+Requires-Dist: omegaconf>=2.3
 Requires-Dist: python-ulid>=3.0.0
-Requires-Dist: rich
-Requires-Dist: typer
+Requires-Dist: rich>=13.9
+Requires-Dist: typer>=0.15
 Description-Content-Type: text/markdown
 
 # Hydraflow

{hydraflow-0.13.0.dist-info → hydraflow-0.14.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hydraflow/__init__.py,sha256=f2KO2iF7um-nNmayNyEr7TWG4UICOXy7YAN1d3qu0OY,936
-hydraflow/cli.py,sha256=nGFrZeQnn1h7lCbVhjBJwdrcDDGoqYpVFbqqwkLMzng,3385
+hydraflow/cli.py,sha256=qzeK6D9yJvtVHvBLf93HJPv19ODv8raLrtNLfDa7IOE,3205
 hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
@@ -13,12 +13,13 @@ hydraflow/entities/run_collection.py,sha256=4sfZWXaS7kqnVCo9GyB3pN6BaSUCkA-ZqSSl
 hydraflow/entities/run_data.py,sha256=lz8HPxG0iz1Jf9FU6HTFW4gcAc3N2pgMyAPqAIK6I74,1644
 hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/executor/conf.py,sha256=2dv6_PlsynRmia-fGZlmBEVt8GopT0f32N13qY7tYnM,402
+hydraflow/executor/aio.py,sha256=xXsmBPIPdBlopv_1h0FdtOvoKUcuW7PQeKCV2d_lN9I,2122
+hydraflow/executor/conf.py,sha256=icGbLDh86KgkyiGXwDoEkmZpgAP3X8Jmu_PYqJoTooY,423
 hydraflow/executor/io.py,sha256=yZMcBVmAbPZZ82cAXhgiJfj9p8WvHmzOCMBg_vtEVek,1509
-hydraflow/executor/job.py,sha256=bmjlqE-cE7lyNFFKj1nUhxiQHKf7DsFkCSTD9iTEQ5I,5606
+hydraflow/executor/job.py,sha256=JX6xX9ffvHB7IiAVIfzVRjjnWKaPDxBgqdZf4ZO14CY,4651
 hydraflow/executor/parser.py,sha256=_Rfund3FDgrXitTt_znsTpgEtMDqZ_ICynaB_Zje14Q,14561
-hydraflow-0.13.0.dist-info/METADATA,sha256=DcBL4IuQHdGIyEXTqFoDGlUdI2sgH0THpTuyFoB3Wg0,4549
-hydraflow-0.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.13.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.13.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.13.0.dist-info/RECORD,,
+hydraflow-0.14.1.dist-info/METADATA,sha256=6qrGuJ9JgAyzMdJ7p1WUTMvJBc7XKoi42ZCCXz__2BE,4566
+hydraflow-0.14.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.14.1.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.14.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.14.1.dist-info/RECORD,,