hydraflow 0.12.3__py3-none-any.whl → 0.12.5__py3-none-any.whl

hydraflow/cli.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import shlex
 from typing import Annotated
 
 import typer
@@ -24,7 +25,13 @@ def run(
     """Run a job."""
 
     from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import multirun, to_text
+    from hydraflow.executor.job import (
+        iter_batches,
+        iter_calls,
+        iter_runs,
+        submit,
+        to_text,
+    )
 
     job = get_job(name)
 
@@ -35,7 +42,24 @@ def run(
     import mlflow
 
     mlflow.set_experiment(job.name)
-    multirun(job)
+
+    if job.submit:
+        funcname, *args = shlex.split(job.submit)
+        submit(funcname, args, iter_batches(job))
+        raise typer.Exit
+
+    if job.run:
+        executable, *args = shlex.split(job.run)
+        it = iter_runs(executable, args, iter_batches(job))
+    elif job.call:
+        funcname, *args = shlex.split(job.call)
+        it = iter_calls(funcname, args, iter_batches(job))
+    else:
+        typer.echo(f"No command found in job: {job.name}.")
+        raise typer.Exit(1)
+
+    for _ in it:
+        pass
 
 
 @app.command()

hydraflow/core/io.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import fnmatch
 import shutil
 import urllib.parse
 import urllib.request
@@ -12,7 +13,7 @@ from hydra.core.hydra_config import HydraConfig
 from omegaconf import DictConfig, ListConfig, OmegaConf
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Iterator
+    from collections.abc import Callable, Iterable, Iterator
 
     from mlflow.entities import Run
 
@@ -152,21 +153,6 @@ def remove_run(run: Run | Iterable[Run]) -> None:
     shutil.rmtree(get_artifact_dir(run).parent)
 
 
-def get_root_dir(uri: str | Path | None = None) -> Path:
-    """Get the root directory for the MLflow tracking server."""
-    import mlflow
-
-    if uri is not None:
-        return Path(uri).absolute()
-
-    uri = mlflow.get_tracking_uri()
-
-    if uri.startswith("file:"):
-        return file_uri_to_path(uri)
-
-    return Path(uri).absolute()
-
-
 def get_experiment_name(path: Path) -> str | None:
     """Get the experiment name from the meta file."""
     metafile = path / "meta.yaml"
@@ -179,47 +165,65 @@ def get_experiment_name(path: Path) -> str | None:
     return None
 
 
+def predicate_experiment_dir(
+    path: Path,
+    experiment_names: list[str] | Callable[[str], bool] | None = None,
+) -> bool:
+    """Predicate an experiment directory based on the path and experiment names."""
+    if not path.is_dir() or path.name in [".trash", "0"]:
+        return False
+
+    name = get_experiment_name(path)
+    if not name:
+        return False
+
+    if experiment_names is None:
+        return True
+
+    if isinstance(experiment_names, list):
+        return any(fnmatch.fnmatch(name, e) for e in experiment_names)
+
+    return experiment_names(name)
+
+
 def iter_experiment_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the experiment directories in the root directory."""
     if isinstance(experiment_names, str):
         experiment_names = [experiment_names]
 
-    root_dir = get_root_dir(root_dir)
-    for path in root_dir.iterdir():
-        if path.is_dir() and path.name not in [".trash", "0"]:
-            if name := get_experiment_name(path):
-                if experiment_names is None or name in experiment_names:
-                    yield path
+    for path in Path(root_dir).iterdir():
+        if predicate_experiment_dir(path, experiment_names):
+            yield path
 
 
 def iter_run_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the run directories in the root directory."""
-    for experiment_dir in iter_experiment_dirs(experiment_names, root_dir):
+    for experiment_dir in iter_experiment_dirs(root_dir, experiment_names):
         for path in experiment_dir.iterdir():
             if path.is_dir() and (path / "artifacts").exists():
                 yield path
 
 
 def iter_artifacts_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifacts directories in the root directory."""
-    for path in iter_run_dirs(experiment_names, root_dir):
+    for path in iter_run_dirs(root_dir, experiment_names):
         yield path / "artifacts"
 
 
 def iter_artifact_paths(
+    root_dir: str | Path,
     artifact_path: str | Path,
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifact paths in the root directory."""
-    for path in iter_artifacts_dirs(experiment_names, root_dir):
+    for path in iter_artifacts_dirs(root_dir, experiment_names):
         yield path / artifact_path

hydraflow/executor/conf.py

@@ -15,6 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
+    submit: str = ""
     with_: str = ""
     steps: list[Step] = field(default_factory=list)
 

hydraflow/executor/job.py

@@ -21,7 +21,7 @@ import importlib
 import shlex
 import subprocess
 import sys
-from subprocess import CalledProcessError
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 import ulid
@@ -29,7 +29,9 @@ import ulid
 from .parser import collect, expand
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Callable, Iterable, Iterator
+    from subprocess import CompletedProcess
+    from typing import Any
 
     from .conf import Job
 
@@ -79,63 +81,81 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
             yield ["--multirun", *args, job_name, sweep_dir, *configs]
 
 
-def multirun(job: Job) -> None:
-    """Execute multiple runs of a job using either shell commands or Python functions.
+@dataclass
+class Run:
+    """An executed run."""
 
-    This function processes a job configuration and executes it in one of two modes:
+    total: int
+    completed: int
+    result: CompletedProcess
 
-    1. Shell command mode (job.run): Executes shell commands with the generated
-       arguments
-    2. Python function mode (job.call): Calls a Python function with the generated
-       arguments
 
-    Args:
-        job (Job): The job configuration containing run parameters and steps.
+@dataclass
+class Call:
+    """An executed call."""
 
-    Raises:
-        RuntimeError: If a shell command fails or if a function call encounters
-            an error.
-        ValueError: If the Python function path is invalid or the function cannot
-            be imported.
+    total: int
+    completed: int
+    result: Any
 
-    """
-    it = iter_batches(job)
 
-    if job.run:
-        base_cmds = shlex.split(job.run)
-        if base_cmds[0] == "python" and sys.platform == "win32":
-            base_cmds[0] = sys.executable
+def iter_runs(
+    executable: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Iterator[Run]:
+    """Execute multiple runs of a job using shell commands."""
+    if executable == "python" and sys.platform == "win32":
+        executable = sys.executable
 
-        for args in it:
-            cmds = [*base_cmds, *args]
-            try:
-                subprocess.run(cmds, check=True)
-            except CalledProcessError as e:
-                msg = f"Command failed with exit code {e.returncode}"
-                raise RuntimeError(msg) from e
+    iterable = list(iterable)
+    total = len(iterable)
 
-    elif job.call:
-        call_name, *base_args = shlex.split(job.call)
+    for completed, args_ in enumerate(iterable, 1):
+        result = subprocess.run([executable, *args, *args_], check=False)
+        yield Run(total, completed, result)
 
-        if "." not in call_name:
-            msg = f"Invalid function path: {call_name}."
-            msg += " Expected format: 'package.module.function'"
-            raise ValueError(msg)
 
-        try:
-            module_name, func_name = call_name.rsplit(".", 1)
-            module = importlib.import_module(module_name)
-            func = getattr(module, func_name)
-        except (ImportError, AttributeError, ModuleNotFoundError) as e:
-            msg = f"Failed to import or find function: {call_name}"
-            raise ValueError(msg) from e
+def iter_calls(
+    funcname: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Iterator[Call]:
+    """Execute multiple calls of a job using Python functions."""
+    func = get_callable(funcname)
 
-        for args in it:
-            try:
-                func([*base_args, *args])
-            except Exception as e:  # noqa: PERF203
-                msg = f"Function call '{job.call}' failed with args: {args}"
-                raise RuntimeError(msg) from e
+    iterable = list(iterable)
+    total = len(iterable)
+
+    for completed, args_ in enumerate(iterable, 1):
+        result = func([*args, *args_])
+        yield Call(total, completed, result)
+
+
+def submit(
+    funcname: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Any:
+    """Submit entire job using Python functions."""
+    func = get_callable(funcname)
+    return func([[*args, *a] for a in iterable])
+
+
+def get_callable(name: str) -> Callable:
+    """Get a callable from a function name."""
+    if "." not in name:
+        msg = f"Invalid function path: {name}."
+        raise ValueError(msg)
+
+    try:
+        module_name, func_name = name.rsplit(".", 1)
+        module = importlib.import_module(module_name)
+        return getattr(module, func_name)
+
+    except (ImportError, AttributeError, ModuleNotFoundError) as e:
+        msg = f"Failed to import or find function: {name}"
+        raise ValueError(msg) from e
 
 
 def to_text(job: Job) -> str:
@@ -165,4 +185,9 @@ def to_text(job: Job) -> str:
         for args in it:
             text += f"args: {args}\n"
 
+    elif job.submit:
+        text = f"submit: {job.submit}\n"
+        for args in it:
+            text += f"args: {args}\n"
+
     return text.rstrip()

{hydraflow-0.12.3.dist-info → hydraflow-0.12.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.12.3
+Version: 0.12.5
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.12.3.dist-info → hydraflow-0.12.5.dist-info}/RECORD

@@ -1,10 +1,10 @@
 hydraflow/__init__.py,sha256=f2KO2iF7um-nNmayNyEr7TWG4UICOXy7YAN1d3qu0OY,936
-hydraflow/cli.py,sha256=clQ7PD_okuGhbrgQ0q8Ldbb8xY3u3He5hI_uBI4h1Q4,1470
+hydraflow/cli.py,sha256=Z9gpuxW6sgGg8lO5nmDE7dToLnNuWVT4nk9j0opqqHs,2075
 hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
 hydraflow/core/context.py,sha256=L4OygMLbITwlWzq17Lh8VoXKKtjOJ3DBEVsBddKPSJ8,4741
-hydraflow/core/io.py,sha256=7rVEcXus0tES2qZIckIOcBxc9Br-nSEdWzGGA7JedB0,6796
+hydraflow/core/io.py,sha256=Tch85xbdRao7rG9BMbRpc2Cq0glC8a8M87QDoyQ81p8,6926
 hydraflow/core/main.py,sha256=dY8uUykS_AbzverrSWkXLyj98TjBPHAiMUf_l5met1U,5162
 hydraflow/core/mlflow.py,sha256=OQJ3f2wkHJRb11ZK__HF4R8FyBEje7-NOqObpoanGhU,5704
 hydraflow/core/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
@@ -13,12 +13,12 @@ hydraflow/entities/run_collection.py,sha256=4sfZWXaS7kqnVCo9GyB3pN6BaSUCkA-ZqSSl
 hydraflow/entities/run_data.py,sha256=lz8HPxG0iz1Jf9FU6HTFW4gcAc3N2pgMyAPqAIK6I74,1644
 hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/executor/conf.py,sha256=2dv6_PlsynRmia-fGZlmBEVt8GopT0f32N13qY7tYnM,402
+hydraflow/executor/conf.py,sha256=icGbLDh86KgkyiGXwDoEkmZpgAP3X8Jmu_PYqJoTooY,423
 hydraflow/executor/io.py,sha256=yZMcBVmAbPZZ82cAXhgiJfj9p8WvHmzOCMBg_vtEVek,1509
-hydraflow/executor/job.py,sha256=IL7ek0Vwa3Bl_gANq0wCbldNCUclo8YBckeEeO6W6xg,4852
+hydraflow/executor/job.py,sha256=Ug_4iVL-rNzMvVjBDecn3FVFGLmRzJEf4Pu6LP_x3LA,4844
 hydraflow/executor/parser.py,sha256=_Rfund3FDgrXitTt_znsTpgEtMDqZ_ICynaB_Zje14Q,14561
-hydraflow-0.12.3.dist-info/METADATA,sha256=Rlwy1HJ4yI5O7bKLSZhZ1JhKIz3tL8Z-TGsREPhfUpw,4549
-hydraflow-0.12.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.12.3.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.12.3.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.12.3.dist-info/RECORD,,
+hydraflow-0.12.5.dist-info/METADATA,sha256=Ztz0IFLx22RTWpPkVguyfUrsaVaZ3WIzyaA1Lwc3sYQ,4549
+hydraflow-0.12.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.12.5.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.12.5.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.12.5.dist-info/RECORD,,