hydraflow 0.12.5__py3-none-any.whl → 0.13.0__py3-none-any.whl

hydraflow/cli.py

@@ -13,53 +13,86 @@ app = typer.Typer(add_completion=False)
 console = Console()
 
 
-@app.command()
+@app.command(context_settings={"ignore_unknown_options": True})
 def run(
     name: Annotated[str, Argument(help="Job name.", show_default=False)],
     *,
+    args: Annotated[
+        list[str] | None,
+        Argument(help="Arguments to pass to the job.", show_default=False),
+    ] = None,
     dry_run: Annotated[
         bool,
-        Option("--dry-run", help="Perform a dry run"),
+        Option("--dry-run", help="Perform a dry run."),
     ] = False,
 ) -> None:
     """Run a job."""
-
     from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import (
-        iter_batches,
-        iter_calls,
-        iter_runs,
-        submit,
-        to_text,
-    )
+    from hydraflow.executor.job import iter_batches, iter_calls, iter_runs
 
+    args = args or []
     job = get_job(name)
 
-    if dry_run:
-        typer.echo(to_text(job))
-        raise typer.Exit
-
-    import mlflow
-
-    mlflow.set_experiment(job.name)
-
-    if job.submit:
-        funcname, *args = shlex.split(job.submit)
-        submit(funcname, args, iter_batches(job))
-        raise typer.Exit
-
     if job.run:
-        executable, *args = shlex.split(job.run)
-        it = iter_runs(executable, args, iter_batches(job))
+        args = [*shlex.split(job.run), *args]
+        it = iter_runs(args, iter_batches(job), dry_run=dry_run)
     elif job.call:
-        funcname, *args = shlex.split(job.call)
-        it = iter_calls(funcname, args, iter_batches(job))
+        args = [*shlex.split(job.call), *args]
+        it = iter_calls(args, iter_batches(job), dry_run=dry_run)
     else:
         typer.echo(f"No command found in job: {job.name}.")
         raise typer.Exit(1)
 
-    for _ in it:
-        pass
+    if not dry_run:
+        import mlflow
+
+        mlflow.set_experiment(job.name)
+
+    for task in it:  # jobs will be executed here
+        if job.run and dry_run:
+            typer.echo(shlex.join(task.args))
+        elif job.call and dry_run:
+            funcname, *args = task.args
+            arg = ", ".join(f"{arg!r}" for arg in args)
+            typer.echo(f"{funcname}([{arg}])")
+
+
+@app.command(context_settings={"ignore_unknown_options": True})
+def submit(
+    name: Annotated[str, Argument(help="Job name.", show_default=False)],
+    *,
+    args: Annotated[
+        list[str] | None,
+        Argument(help="Arguments to pass to the job.", show_default=False),
+    ] = None,
+    dry_run: Annotated[
+        bool,
+        Option("--dry-run", help="Perform a dry run."),
+    ] = False,
+) -> None:
+    """Submit a job."""
+    from hydraflow.executor.io import get_job
+    from hydraflow.executor.job import iter_batches, submit
+
+    args = args or []
+    job = get_job(name)
+
+    if not job.run:
+        typer.echo(f"No run found in job: {job.name}.")
+        raise typer.Exit(1)
+
+    if not dry_run:
+        import mlflow
+
+        mlflow.set_experiment(job.name)
+
+    args = [*shlex.split(job.run), *args]
+    result = submit(args, iter_batches(job), dry_run=dry_run)
+
+    if dry_run and isinstance(result, tuple):
+        for line in result[1].splitlines():
+            args = shlex.split(line)
+            typer.echo(shlex.join([*result[0][:-1], *args]))
 
 
 @app.command()

hydraflow/executor/conf.py

@@ -15,7 +15,6 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
-    submit: str = ""
     with_: str = ""
     steps: list[Step] = field(default_factory=list)
 

hydraflow/executor/job.py

@@ -22,7 +22,10 @@ import shlex
 import subprocess
 import sys
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from pathlib import Path
+from subprocess import CompletedProcess
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, overload
 
 import ulid
 
@@ -82,29 +85,49 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
 
 @dataclass
-class Run:
-    """An executed run."""
+class Task:
+    """An executed task."""
 
+    args: list[str]
     total: int
     completed: int
+
+
+@dataclass
+class Run(Task):
+    """An executed run."""
+
     result: CompletedProcess
 
 
 @dataclass
-class Call:
+class Call(Task):
     """An executed call."""
 
-    total: int
-    completed: int
     result: Any
 
 
+@overload
+def iter_runs(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Run]: ...
+
+
+@overload
+def iter_runs(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Run]: ...
+
+
 def iter_runs(
-    executable: str,
     args: list[str],
     iterable: Iterable[list[str]],
-) -> Iterator[Run]:
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Run]:
     """Execute multiple runs of a job using shell commands."""
+    executable, *args = args
     if executable == "python" and sys.platform == "win32":
         executable = sys.executable
 
@@ -112,34 +135,75 @@ def iter_runs(
     total = len(iterable)
 
     for completed, args_ in enumerate(iterable, 1):
-        result = subprocess.run([executable, *args, *args_], check=False)
-        yield Run(total, completed, result)
+        cmd = [executable, *args, *args_]
+        if dry_run:
+            yield Task(cmd, total, completed)
+        else:
+            result = subprocess.run(cmd, check=False)
+            yield Run(cmd, total, completed, result)
+
+
+@overload
+def iter_calls(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Call]: ...
+
+
+@overload
+def iter_calls(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Call]: ...
 
 
 def iter_calls(
-    funcname: str,
     args: list[str],
     iterable: Iterable[list[str]],
-) -> Iterator[Call]:
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Call]:
     """Execute multiple calls of a job using Python functions."""
+    funcname, *args = args
     func = get_callable(funcname)
 
     iterable = list(iterable)
     total = len(iterable)
 
     for completed, args_ in enumerate(iterable, 1):
-        result = func([*args, *args_])
-        yield Call(total, completed, result)
+        cmd = [funcname, *args, *args_]
+        if dry_run:
+            yield Task(cmd, total, completed)
+        else:
+            result = func([*args, *args_])
+            yield Call(cmd, total, completed, result)
 
 
 def submit(
-    funcname: str,
     args: list[str],
     iterable: Iterable[list[str]],
-) -> Any:
-    """Submit entire job using Python functions."""
-    func = get_callable(funcname)
-    return func([[*args, *a] for a in iterable])
+    *,
+    dry_run: bool = False,
+) -> CompletedProcess | tuple[list[str], str]:
+    """Submit entire job using a shell command."""
+    executable, *args = args
+    if executable == "python" and sys.platform == "win32":
+        executable = sys.executable
+
+    temp = NamedTemporaryFile(dir=Path.cwd(), delete=False)  # for Windows
+    file = Path(temp.name)
+    temp.close()
+
+    text = "\n".join(shlex.join(args) for args in iterable)
+    file.write_text(text)
+    cmd = [executable, *args, file.as_posix()]
+
+    try:
+        if dry_run:
+            return cmd, text
+        return subprocess.run(cmd, check=False)
+
+    finally:
+        file.unlink(missing_ok=True)
 
 
 def get_callable(name: str) -> Callable:
@@ -156,38 +220,3 @@ def get_callable(name: str) -> Callable:
     except (ImportError, AttributeError, ModuleNotFoundError) as e:
         msg = f"Failed to import or find function: {name}"
         raise ValueError(msg) from e
-
-
-def to_text(job: Job) -> str:
-    """Convert the job configuration to a string.
-
-    This function returns the job configuration for a given job.
-
-    Args:
-        job (Job): The job configuration to show.
-
-    Returns:
-        str: The job configuration.
-
-    """
-    text = ""
-
-    it = iter_batches(job)
-
-    if job.run:
-        base_cmds = shlex.split(job.run)
-        for args in it:
-            cmds = " ".join([*base_cmds, *args])
-            text += f"{cmds}\n"
-
-    elif job.call:
-        text = f"call: {job.call}\n"
-        for args in it:
-            text += f"args: {args}\n"
-
-    elif job.submit:
-        text = f"submit: {job.submit}\n"
-        for args in it:
-            text += f"args: {args}\n"
-
-    return text.rstrip()

{hydraflow-0.12.5.dist-info → hydraflow-0.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.12.5
+Version: 0.13.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.12.5.dist-info → hydraflow-0.13.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hydraflow/__init__.py,sha256=f2KO2iF7um-nNmayNyEr7TWG4UICOXy7YAN1d3qu0OY,936
-hydraflow/cli.py,sha256=Z9gpuxW6sgGg8lO5nmDE7dToLnNuWVT4nk9j0opqqHs,2075
+hydraflow/cli.py,sha256=nGFrZeQnn1h7lCbVhjBJwdrcDDGoqYpVFbqqwkLMzng,3385
 hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
@@ -13,12 +13,12 @@ hydraflow/entities/run_collection.py,sha256=4sfZWXaS7kqnVCo9GyB3pN6BaSUCkA-ZqSSl
 hydraflow/entities/run_data.py,sha256=lz8HPxG0iz1Jf9FU6HTFW4gcAc3N2pgMyAPqAIK6I74,1644
 hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/executor/conf.py,sha256=icGbLDh86KgkyiGXwDoEkmZpgAP3X8Jmu_PYqJoTooY,423
+hydraflow/executor/conf.py,sha256=2dv6_PlsynRmia-fGZlmBEVt8GopT0f32N13qY7tYnM,402
 hydraflow/executor/io.py,sha256=yZMcBVmAbPZZ82cAXhgiJfj9p8WvHmzOCMBg_vtEVek,1509
-hydraflow/executor/job.py,sha256=Ug_4iVL-rNzMvVjBDecn3FVFGLmRzJEf4Pu6LP_x3LA,4844
+hydraflow/executor/job.py,sha256=bmjlqE-cE7lyNFFKj1nUhxiQHKf7DsFkCSTD9iTEQ5I,5606
 hydraflow/executor/parser.py,sha256=_Rfund3FDgrXitTt_znsTpgEtMDqZ_ICynaB_Zje14Q,14561
-hydraflow-0.12.5.dist-info/METADATA,sha256=Ztz0IFLx22RTWpPkVguyfUrsaVaZ3WIzyaA1Lwc3sYQ,4549
-hydraflow-0.12.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.12.5.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.12.5.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.12.5.dist-info/RECORD,,
+hydraflow-0.13.0.dist-info/METADATA,sha256=DcBL4IuQHdGIyEXTqFoDGlUdI2sgH0THpTuyFoB3Wg0,4549
+hydraflow-0.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.13.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.13.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.13.0.dist-info/RECORD,,