hydraflow 0.12.4__py3-none-any.whl → 0.13.0__py3-none-any.whl

hydraflow/cli.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import shlex
 from typing import Annotated
 
 import typer
@@ -12,30 +13,86 @@ app = typer.Typer(add_completion=False)
 console = Console()
 
 
-@app.command()
+@app.command(context_settings={"ignore_unknown_options": True})
 def run(
     name: Annotated[str, Argument(help="Job name.", show_default=False)],
     *,
+    args: Annotated[
+        list[str] | None,
+        Argument(help="Arguments to pass to the job.", show_default=False),
+    ] = None,
     dry_run: Annotated[
         bool,
-        Option("--dry-run", help="Perform a dry run"),
+        Option("--dry-run", help="Perform a dry run."),
     ] = False,
 ) -> None:
     """Run a job."""
+    from hydraflow.executor.io import get_job
+    from hydraflow.executor.job import iter_batches, iter_calls, iter_runs
+
+    args = args or []
+    job = get_job(name)
+
+    if job.run:
+        args = [*shlex.split(job.run), *args]
+        it = iter_runs(args, iter_batches(job), dry_run=dry_run)
+    elif job.call:
+        args = [*shlex.split(job.call), *args]
+        it = iter_calls(args, iter_batches(job), dry_run=dry_run)
+    else:
+        typer.echo(f"No command found in job: {job.name}.")
+        raise typer.Exit(1)
+
+    if not dry_run:
+        import mlflow
+
+        mlflow.set_experiment(job.name)
+
+    for task in it:  # jobs will be executed here
+        if job.run and dry_run:
+            typer.echo(shlex.join(task.args))
+        elif job.call and dry_run:
+            funcname, *args = task.args
+            arg = ", ".join(f"{arg!r}" for arg in args)
+            typer.echo(f"{funcname}([{arg}])")
+
 
+@app.command(context_settings={"ignore_unknown_options": True})
+def submit(
+    name: Annotated[str, Argument(help="Job name.", show_default=False)],
+    *,
+    args: Annotated[
+        list[str] | None,
+        Argument(help="Arguments to pass to the job.", show_default=False),
+    ] = None,
+    dry_run: Annotated[
+        bool,
+        Option("--dry-run", help="Perform a dry run."),
+    ] = False,
+) -> None:
+    """Submit a job."""
     from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import multirun, to_text
+    from hydraflow.executor.job import iter_batches, submit
 
+    args = args or []
     job = get_job(name)
 
-    if dry_run:
-        typer.echo(to_text(job))
-        raise typer.Exit
+    if not job.run:
+        typer.echo(f"No run found in job: {job.name}.")
+        raise typer.Exit(1)
+
+    if not dry_run:
+        import mlflow
+
+        mlflow.set_experiment(job.name)
 
-    import mlflow
+    args = [*shlex.split(job.run), *args]
+    result = submit(args, iter_batches(job), dry_run=dry_run)
 
-    mlflow.set_experiment(job.name)
-    multirun(job)
+    if dry_run and isinstance(result, tuple):
+        for line in result[1].splitlines():
+            args = shlex.split(line)
+            typer.echo(shlex.join([*result[0][:-1], *args]))
 
 
 @app.command()

hydraflow/core/io.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import fnmatch
 import shutil
 import urllib.parse
 import urllib.request
@@ -152,21 +153,6 @@ def remove_run(run: Run | Iterable[Run]) -> None:
     shutil.rmtree(get_artifact_dir(run).parent)
 
 
-def get_root_dir(uri: str | Path | None = None) -> Path:
-    """Get the root directory for the MLflow tracking server."""
-    import mlflow
-
-    if uri is not None:
-        return Path(uri).absolute()
-
-    uri = mlflow.get_tracking_uri()
-
-    if uri.startswith("file:"):
-        return file_uri_to_path(uri)
-
-    return Path(uri).absolute()
-
-
 def get_experiment_name(path: Path) -> str | None:
     """Get the experiment name from the meta file."""
     metafile = path / "meta.yaml"
@@ -195,50 +181,49 @@ def predicate_experiment_dir(
         return True
 
     if isinstance(experiment_names, list):
-        return name in experiment_names
+        return any(fnmatch.fnmatch(name, e) for e in experiment_names)
 
     return experiment_names(name)
 
 
 def iter_experiment_dirs(
+    root_dir: str | Path,
     experiment_names: str | list[str] | Callable[[str], bool] | None = None,
-    root_dir: str | Path | None = None,
 ) -> Iterator[Path]:
     """Iterate over the experiment directories in the root directory."""
     if isinstance(experiment_names, str):
         experiment_names = [experiment_names]
 
-    root_dir = get_root_dir(root_dir)
-    for path in root_dir.iterdir():
+    for path in Path(root_dir).iterdir():
         if predicate_experiment_dir(path, experiment_names):
             yield path
 
 
 def iter_run_dirs(
+    root_dir: str | Path,
     experiment_names: str | list[str] | Callable[[str], bool] | None = None,
-    root_dir: str | Path | None = None,
 ) -> Iterator[Path]:
     """Iterate over the run directories in the root directory."""
-    for experiment_dir in iter_experiment_dirs(experiment_names, root_dir):
+    for experiment_dir in iter_experiment_dirs(root_dir, experiment_names):
         for path in experiment_dir.iterdir():
             if path.is_dir() and (path / "artifacts").exists():
                 yield path
 
 
 def iter_artifacts_dirs(
+    root_dir: str | Path,
     experiment_names: str | list[str] | Callable[[str], bool] | None = None,
-    root_dir: str | Path | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifacts directories in the root directory."""
-    for path in iter_run_dirs(experiment_names, root_dir):
+    for path in iter_run_dirs(root_dir, experiment_names):
         yield path / "artifacts"
 
 
 def iter_artifact_paths(
+    root_dir: str | Path,
     artifact_path: str | Path,
     experiment_names: str | list[str] | Callable[[str], bool] | None = None,
-    root_dir: str | Path | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifact paths in the root directory."""
-    for path in iter_artifacts_dirs(experiment_names, root_dir):
+    for path in iter_artifacts_dirs(root_dir, experiment_names):
         yield path / artifact_path

hydraflow/executor/job.py

@@ -21,15 +21,20 @@ import importlib
 import shlex
 import subprocess
 import sys
-from subprocess import CalledProcessError
-from typing import TYPE_CHECKING
+from dataclasses import dataclass
+from pathlib import Path
+from subprocess import CompletedProcess
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, overload
 
 import ulid
 
 from .parser import collect, expand
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Callable, Iterable, Iterator
+    from subprocess import CompletedProcess
+    from typing import Any
 
     from .conf import Job
 
@@ -79,90 +84,139 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
             yield ["--multirun", *args, job_name, sweep_dir, *configs]
 
 
-def multirun(job: Job) -> None:
-    """Execute multiple runs of a job using either shell commands or Python functions.
+@dataclass
+class Task:
+    """An executed task."""
 
-    This function processes a job configuration and executes it in one of two modes:
+    args: list[str]
+    total: int
+    completed: int
 
-    1. Shell command mode (job.run): Executes shell commands with the generated
-       arguments
-    2. Python function mode (job.call): Calls a Python function with the generated
-       arguments
 
-    Args:
-        job (Job): The job configuration containing run parameters and steps.
+@dataclass
+class Run(Task):
+    """An executed run."""
 
-    Raises:
-        RuntimeError: If a shell command fails or if a function call encounters
-            an error.
-        ValueError: If the Python function path is invalid or the function cannot
-            be imported.
+    result: CompletedProcess
 
-    """
-    it = iter_batches(job)
-
-    if job.run:
-        base_cmds = shlex.split(job.run)
-        if base_cmds[0] == "python" and sys.platform == "win32":
-            base_cmds[0] = sys.executable
-
-        for args in it:
-            cmds = [*base_cmds, *args]
-            try:
-                subprocess.run(cmds, check=True)
-            except CalledProcessError as e:
-                msg = f"Command failed with exit code {e.returncode}"
-                raise RuntimeError(msg) from e
-
-    elif job.call:
-        call_name, *base_args = shlex.split(job.call)
-
-        if "." not in call_name:
-            msg = f"Invalid function path: {call_name}."
-            msg += " Expected format: 'package.module.function'"
-            raise ValueError(msg)
-
-        try:
-            module_name, func_name = call_name.rsplit(".", 1)
-            module = importlib.import_module(module_name)
-            func = getattr(module, func_name)
-        except (ImportError, AttributeError, ModuleNotFoundError) as e:
-            msg = f"Failed to import or find function: {call_name}"
-            raise ValueError(msg) from e
-
-        for args in it:
-            try:
-                func([*base_args, *args])
-            except Exception as e:  # noqa: PERF203
-                msg = f"Function call '{job.call}' failed with args: {args}"
-                raise RuntimeError(msg) from e
-
-
-def to_text(job: Job) -> str:
-    """Convert the job configuration to a string.
-
-    This function returns the job configuration for a given job.
 
-    Args:
-        job (Job): The job configuration to show.
+@dataclass
+class Call(Task):
+    """An executed call."""
 
-    Returns:
-        str: The job configuration.
+    result: Any
 
-    """
-    text = ""
 
-    it = iter_batches(job)
+@overload
+def iter_runs(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Run]: ...
+
+
+@overload
+def iter_runs(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Run]: ...
+
+
+def iter_runs(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Run]:
+    """Execute multiple runs of a job using shell commands."""
+    executable, *args = args
+    if executable == "python" and sys.platform == "win32":
+        executable = sys.executable
+
+    iterable = list(iterable)
+    total = len(iterable)
+
+    for completed, args_ in enumerate(iterable, 1):
+        cmd = [executable, *args, *args_]
+        if dry_run:
+            yield Task(cmd, total, completed)
+        else:
+            result = subprocess.run(cmd, check=False)
+            yield Run(cmd, total, completed, result)
+
+
+@overload
+def iter_calls(args: list[str], iterable: Iterable[list[str]]) -> Iterator[Call]: ...
+
+
+@overload
+def iter_calls(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Call]: ...
+
+
+def iter_calls(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> Iterator[Task | Call]:
+    """Execute multiple calls of a job using Python functions."""
+    funcname, *args = args
+    func = get_callable(funcname)
+
+    iterable = list(iterable)
+    total = len(iterable)
 
-    if job.run:
-        base_cmds = shlex.split(job.run)
-        for args in it:
-            cmds = " ".join([*base_cmds, *args])
-            text += f"{cmds}\n"
+    for completed, args_ in enumerate(iterable, 1):
+        cmd = [funcname, *args, *args_]
+        if dry_run:
+            yield Task(cmd, total, completed)
+        else:
+            result = func([*args, *args_])
+            yield Call(cmd, total, completed, result)
 
-    elif job.call:
-        text = f"call: {job.call}\n"
-        for args in it:
-            text += f"args: {args}\n"
 
-    return text.rstrip()
+def submit(
+    args: list[str],
+    iterable: Iterable[list[str]],
+    *,
+    dry_run: bool = False,
+) -> CompletedProcess | tuple[list[str], str]:
+    """Submit entire job using a shell command."""
+    executable, *args = args
+    if executable == "python" and sys.platform == "win32":
+        executable = sys.executable
+
+    temp = NamedTemporaryFile(dir=Path.cwd(), delete=False)  # for Windows
+    file = Path(temp.name)
+    temp.close()
+
+    text = "\n".join(shlex.join(args) for args in iterable)
+    file.write_text(text)
+    cmd = [executable, *args, file.as_posix()]
+
+    try:
+        if dry_run:
+            return cmd, text
+        return subprocess.run(cmd, check=False)
+
+    finally:
+        file.unlink(missing_ok=True)
+
+
+def get_callable(name: str) -> Callable:
+    """Get a callable from a function name."""
+    if "." not in name:
+        msg = f"Invalid function path: {name}."
+        raise ValueError(msg)
+
+    try:
+        module_name, func_name = name.rsplit(".", 1)
+        module = importlib.import_module(module_name)
+        return getattr(module, func_name)
+
+    except (ImportError, AttributeError, ModuleNotFoundError) as e:
+        msg = f"Failed to import or find function: {name}"
+        raise ValueError(msg) from e

{hydraflow-0.12.4.dist-info → hydraflow-0.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.12.4
+Version: 0.13.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.12.4.dist-info → hydraflow-0.13.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 hydraflow/__init__.py,sha256=f2KO2iF7um-nNmayNyEr7TWG4UICOXy7YAN1d3qu0OY,936
-hydraflow/cli.py,sha256=clQ7PD_okuGhbrgQ0q8Ldbb8xY3u3He5hI_uBI4h1Q4,1470
+hydraflow/cli.py,sha256=nGFrZeQnn1h7lCbVhjBJwdrcDDGoqYpVFbqqwkLMzng,3385
 hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
 hydraflow/core/context.py,sha256=L4OygMLbITwlWzq17Lh8VoXKKtjOJ3DBEVsBddKPSJ8,4741
-hydraflow/core/io.py,sha256=6ZQYqxPUkIinFYqSQXJPzTSnuhfP1KfFchTNeNn-g6A,7311
+hydraflow/core/io.py,sha256=Tch85xbdRao7rG9BMbRpc2Cq0glC8a8M87QDoyQ81p8,6926
 hydraflow/core/main.py,sha256=dY8uUykS_AbzverrSWkXLyj98TjBPHAiMUf_l5met1U,5162
 hydraflow/core/mlflow.py,sha256=OQJ3f2wkHJRb11ZK__HF4R8FyBEje7-NOqObpoanGhU,5704
 hydraflow/core/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
@@ -15,10 +15,10 @@ hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmH
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/executor/conf.py,sha256=2dv6_PlsynRmia-fGZlmBEVt8GopT0f32N13qY7tYnM,402
 hydraflow/executor/io.py,sha256=yZMcBVmAbPZZ82cAXhgiJfj9p8WvHmzOCMBg_vtEVek,1509
-hydraflow/executor/job.py,sha256=IL7ek0Vwa3Bl_gANq0wCbldNCUclo8YBckeEeO6W6xg,4852
+hydraflow/executor/job.py,sha256=bmjlqE-cE7lyNFFKj1nUhxiQHKf7DsFkCSTD9iTEQ5I,5606
 hydraflow/executor/parser.py,sha256=_Rfund3FDgrXitTt_znsTpgEtMDqZ_ICynaB_Zje14Q,14561
-hydraflow-0.12.4.dist-info/METADATA,sha256=CT9bb--1HhyO_tcahfCTzSUBH2nWt7i2lY2pS3Dac38,4549
-hydraflow-0.12.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.12.4.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.12.4.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.12.4.dist-info/RECORD,,
+hydraflow-0.13.0.dist-info/METADATA,sha256=DcBL4IuQHdGIyEXTqFoDGlUdI2sgH0THpTuyFoB3Wg0,4549
+hydraflow-0.13.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.13.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.13.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.13.0.dist-info/RECORD,,