hydraflow 0.12.3__tar.gz → 0.12.5__tar.gz

{hydraflow-0.12.3 → hydraflow-0.12.5}/.github/workflows/ci.yaml

@@ -14,7 +14,7 @@ env:
   FORCE_COLOR: "1"
 
 jobs:
-  run:
+  ci:
     name: Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -29,7 +29,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          allow-prereleases: true
       - name: Install uv and ruff
         run: pip install uv ruff
       - name: Install the project

{hydraflow-0.12.3 → hydraflow-0.12.5}/.github/workflows/docs.yaml

@@ -1,25 +1,29 @@
 name: Documentation
+
 on:
   push:
     branches: [main]
-    tags: ["*"]
-permissions:
-  contents: write
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+"
+
 jobs:
-  deploy:
-    name: Documentation
+  docs:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
       - name: Configure Git Credentials
         run: |
           git config user.name github-actions[bot]
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
-      - name: Set up Python 3.11
+      - name: Set up Python 3.13
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
-      - name: Install package
-        run: pip install -e . mkapi markdown-exec[ansi]
+          python-version: 3.13
+      - name: Install uv
+        run: pip install uv
+      - name: Install the project
+        run: uv sync --group docs
       - name: Deploy documentation
-        run: mkdocs gh-deploy --force
+        run: uv run mkdocs gh-deploy --force

hydraflow-0.12.5/.github/workflows/publish.yaml

@@ -0,0 +1,37 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Verify version match
+        run: |
+          TAG=${GITHUB_REF#refs/tags/}
+          PYPROJECT_VERSION=$(grep -m 1 "version = " pyproject.toml | cut -d'"' -f2)
+          if [ "$TAG" != "$PYPROJECT_VERSION" ]; then
+            echo "Error: Git tag ($TAG) does not match pyproject.toml version ($PYPROJECT_VERSION)"
+            exit 1
+          fi
+          echo "Version verified: $TAG"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.13
+      - name: Install uv
+        run: pip install uv
+      - name: Install the project
+        run: uv sync
+      - name: Build the project
+        run: uv build --no-sources
+      - name: Upload the project to PyPI
+        run: uv publish

{hydraflow-0.12.3 → hydraflow-0.12.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.12.3
+Version: 0.12.5
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.12.3 → hydraflow-0.12.5}/docs/index.md

@@ -1,10 +1,10 @@
-# Hydraflow Documentation
-
-Hydraflow integrates [Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/)
-to manage and track machine learning experiments.
-
-## Installation
-
-```bash
-pip install hydraflow
+# Hydraflow Documentation
+
+Hydraflow integrates [Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/)
+to manage and track machine learning experiments.
+
+## Installation
+
+```bash
+pip install hydraflow
 ```

{hydraflow-0.12.3 → hydraflow-0.12.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.12.3"
+version = "0.12.5"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = { file = "LICENSE" }

{hydraflow-0.12.3 → hydraflow-0.12.5}/src/hydraflow/cli.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import shlex
 from typing import Annotated
 
 import typer
@@ -24,7 +25,13 @@ def run(
     """Run a job."""
 
     from hydraflow.executor.io import get_job
-    from hydraflow.executor.job import multirun, to_text
+    from hydraflow.executor.job import (
+        iter_batches,
+        iter_calls,
+        iter_runs,
+        submit,
+        to_text,
+    )
 
     job = get_job(name)
 
@@ -35,7 +42,24 @@ def run(
     import mlflow
 
     mlflow.set_experiment(job.name)
-    multirun(job)
+
+    if job.submit:
+        funcname, *args = shlex.split(job.submit)
+        submit(funcname, args, iter_batches(job))
+        raise typer.Exit
+
+    if job.run:
+        executable, *args = shlex.split(job.run)
+        it = iter_runs(executable, args, iter_batches(job))
+    elif job.call:
+        funcname, *args = shlex.split(job.call)
+        it = iter_calls(funcname, args, iter_batches(job))
+    else:
+        typer.echo(f"No command found in job: {job.name}.")
+        raise typer.Exit(1)
+
+    for _ in it:
+        pass
 
 
 @app.command()

{hydraflow-0.12.3 → hydraflow-0.12.5}/src/hydraflow/core/io.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import fnmatch
 import shutil
 import urllib.parse
 import urllib.request
@@ -12,7 +13,7 @@ from hydra.core.hydra_config import HydraConfig
 from omegaconf import DictConfig, ListConfig, OmegaConf
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Iterator
+    from collections.abc import Callable, Iterable, Iterator
 
     from mlflow.entities import Run
 
@@ -152,21 +153,6 @@ def remove_run(run: Run | Iterable[Run]) -> None:
     shutil.rmtree(get_artifact_dir(run).parent)
 
 
-def get_root_dir(uri: str | Path | None = None) -> Path:
-    """Get the root directory for the MLflow tracking server."""
-    import mlflow
-
-    if uri is not None:
-        return Path(uri).absolute()
-
-    uri = mlflow.get_tracking_uri()
-
-    if uri.startswith("file:"):
-        return file_uri_to_path(uri)
-
-    return Path(uri).absolute()
-
-
 def get_experiment_name(path: Path) -> str | None:
     """Get the experiment name from the meta file."""
     metafile = path / "meta.yaml"
@@ -179,47 +165,65 @@ def get_experiment_name(path: Path) -> str | None:
     return None
 
 
+def predicate_experiment_dir(
+    path: Path,
+    experiment_names: list[str] | Callable[[str], bool] | None = None,
+) -> bool:
+    """Predicate an experiment directory based on the path and experiment names."""
+    if not path.is_dir() or path.name in [".trash", "0"]:
+        return False
+
+    name = get_experiment_name(path)
+    if not name:
+        return False
+
+    if experiment_names is None:
+        return True
+
+    if isinstance(experiment_names, list):
+        return any(fnmatch.fnmatch(name, e) for e in experiment_names)
+
+    return experiment_names(name)
+
+
 def iter_experiment_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the experiment directories in the root directory."""
     if isinstance(experiment_names, str):
         experiment_names = [experiment_names]
 
-    root_dir = get_root_dir(root_dir)
-    for path in root_dir.iterdir():
-        if path.is_dir() and path.name not in [".trash", "0"]:
-            if name := get_experiment_name(path):
-                if experiment_names is None or name in experiment_names:
-                    yield path
+    for path in Path(root_dir).iterdir():
+        if predicate_experiment_dir(path, experiment_names):
+            yield path
 
 
 def iter_run_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the run directories in the root directory."""
-    for experiment_dir in iter_experiment_dirs(experiment_names, root_dir):
+    for experiment_dir in iter_experiment_dirs(root_dir, experiment_names):
         for path in experiment_dir.iterdir():
             if path.is_dir() and (path / "artifacts").exists():
                 yield path
 
 
 def iter_artifacts_dirs(
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    root_dir: str | Path,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifacts directories in the root directory."""
-    for path in iter_run_dirs(experiment_names, root_dir):
+    for path in iter_run_dirs(root_dir, experiment_names):
         yield path / "artifacts"
 
 
 def iter_artifact_paths(
+    root_dir: str | Path,
     artifact_path: str | Path,
-    experiment_names: str | list[str] | None = None,
-    root_dir: str | Path | None = None,
+    experiment_names: str | list[str] | Callable[[str], bool] | None = None,
 ) -> Iterator[Path]:
     """Iterate over the artifact paths in the root directory."""
-    for path in iter_artifacts_dirs(experiment_names, root_dir):
+    for path in iter_artifacts_dirs(root_dir, experiment_names):
         yield path / artifact_path

{hydraflow-0.12.3 → hydraflow-0.12.5}/src/hydraflow/executor/conf.py

@@ -15,6 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
+    submit: str = ""
     with_: str = ""
     steps: list[Step] = field(default_factory=list)
 

{hydraflow-0.12.3 → hydraflow-0.12.5}/src/hydraflow/executor/job.py

@@ -21,7 +21,7 @@ import importlib
 import shlex
 import subprocess
 import sys
-from subprocess import CalledProcessError
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 import ulid
@@ -29,7 +29,9 @@ import ulid
 from .parser import collect, expand
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
+    from collections.abc import Callable, Iterable, Iterator
+    from subprocess import CompletedProcess
+    from typing import Any
 
     from .conf import Job
 
@@ -79,63 +81,81 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
             yield ["--multirun", *args, job_name, sweep_dir, *configs]
 
 
-def multirun(job: Job) -> None:
-    """Execute multiple runs of a job using either shell commands or Python functions.
+@dataclass
+class Run:
+    """An executed run."""
 
-    This function processes a job configuration and executes it in one of two modes:
+    total: int
+    completed: int
+    result: CompletedProcess
 
-    1. Shell command mode (job.run): Executes shell commands with the generated
-       arguments
-    2. Python function mode (job.call): Calls a Python function with the generated
-       arguments
 
-    Args:
-        job (Job): The job configuration containing run parameters and steps.
+@dataclass
+class Call:
+    """An executed call."""
 
-    Raises:
-        RuntimeError: If a shell command fails or if a function call encounters
-            an error.
-        ValueError: If the Python function path is invalid or the function cannot
-            be imported.
+    total: int
+    completed: int
+    result: Any
 
-    """
-    it = iter_batches(job)
 
-    if job.run:
-        base_cmds = shlex.split(job.run)
-        if base_cmds[0] == "python" and sys.platform == "win32":
-            base_cmds[0] = sys.executable
+def iter_runs(
+    executable: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Iterator[Run]:
+    """Execute multiple runs of a job using shell commands."""
+    if executable == "python" and sys.platform == "win32":
+        executable = sys.executable
 
-        for args in it:
-            cmds = [*base_cmds, *args]
-            try:
-                subprocess.run(cmds, check=True)
-            except CalledProcessError as e:
-                msg = f"Command failed with exit code {e.returncode}"
-                raise RuntimeError(msg) from e
+    iterable = list(iterable)
+    total = len(iterable)
 
-    elif job.call:
-        call_name, *base_args = shlex.split(job.call)
+    for completed, args_ in enumerate(iterable, 1):
+        result = subprocess.run([executable, *args, *args_], check=False)
+        yield Run(total, completed, result)
 
-        if "." not in call_name:
-            msg = f"Invalid function path: {call_name}."
-            msg += " Expected format: 'package.module.function'"
-            raise ValueError(msg)
 
-        try:
-            module_name, func_name = call_name.rsplit(".", 1)
-            module = importlib.import_module(module_name)
-            func = getattr(module, func_name)
-        except (ImportError, AttributeError, ModuleNotFoundError) as e:
-            msg = f"Failed to import or find function: {call_name}"
-            raise ValueError(msg) from e
+def iter_calls(
+    funcname: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Iterator[Call]:
+    """Execute multiple calls of a job using Python functions."""
+    func = get_callable(funcname)
 
-        for args in it:
-            try:
-                func([*base_args, *args])
-            except Exception as e:  # noqa: PERF203
-                msg = f"Function call '{job.call}' failed with args: {args}"
-                raise RuntimeError(msg) from e
+    iterable = list(iterable)
+    total = len(iterable)
+
+    for completed, args_ in enumerate(iterable, 1):
+        result = func([*args, *args_])
+        yield Call(total, completed, result)
+
+
+def submit(
+    funcname: str,
+    args: list[str],
+    iterable: Iterable[list[str]],
+) -> Any:
+    """Submit entire job using Python functions."""
+    func = get_callable(funcname)
+    return func([[*args, *a] for a in iterable])
+
+
+def get_callable(name: str) -> Callable:
+    """Get a callable from a function name."""
+    if "." not in name:
+        msg = f"Invalid function path: {name}."
+        raise ValueError(msg)
+
+    try:
+        module_name, func_name = name.rsplit(".", 1)
+        module = importlib.import_module(module_name)
+        return getattr(module, func_name)
+
+    except (ImportError, AttributeError, ModuleNotFoundError) as e:
+        msg = f"Failed to import or find function: {name}"
+        raise ValueError(msg) from e
 
 
 def to_text(job: Job) -> str:
@@ -165,4 +185,9 @@ def to_text(job: Job) -> str:
         for args in it:
             text += f"args: {args}\n"
 
+    elif job.submit:
+        text = f"submit: {job.submit}\n"
+        for args in it:
+            text += f"args: {args}\n"
+
     return text.rstrip()

{hydraflow-0.12.3 → hydraflow-0.12.5}/tests/cli/hydraflow.yaml

@@ -20,3 +20,21 @@ jobs:
       - batch: name=b
         args: count=11:14
         with: hydra/launcher=joblib hydra.launcher.n_jobs=4
+  echo:
+    call: typer.echo a b c
+    steps:
+      - batch: name=a,b
+        args: count=1:3
+      - batch: name=c,d
+        args: count=4:6
+  submit:
+    submit: typer.echo
+    steps:
+      - batch: name=a
+        args: count=1,3
+      - batch: name=b
+        args: count=5,7
+  error:
+    steps:
+      - batch: name=a
+        args: count=1:3

{hydraflow-0.12.3 → hydraflow-0.12.5}/tests/cli/test_run.py

@@ -61,3 +61,33 @@ def test_run_parallel():
     assert result.exit_code == 0
     run_ids = hydraflow.list_run_ids("parallel")
     assert len(run_ids) == 8
+
+
+@pytest.mark.xdist_group(name="group4")
+def test_run_echo():
+    result = runner.invoke(app, ["run", "echo"])
+    assert result.exit_code == 0
+    out = result.stdout
+    lines = out.splitlines()
+    assert len(lines) == 5
+    assert "['a', 'b', 'c', '--multirun', 'name=a', 'count=1,2,3'" in lines[1]
+    assert "['a', 'b', 'c', '--multirun', 'name=b', 'count=1,2,3'" in lines[2]
+    assert "['a', 'b', 'c', '--multirun', 'name=c', 'count=4,5,6'" in lines[3]
+    assert "['a', 'b', 'c', '--multirun', 'name=d', 'count=4,5,6'" in lines[4]
+
+
+@pytest.mark.xdist_group(name="group4")
+def test_run_submit():
+    result = runner.invoke(app, ["run", "submit"])
+    assert result.exit_code == 0
+    out = result.stdout
+    lines = out.splitlines()
+    assert len(lines) == 2
+    assert "[['--multirun', 'name=a', 'count=1,3', 'hydra.job.name=submit'" in lines[1]
+
+
+@pytest.mark.xdist_group(name="group5")
+def test_run_error():
+    result = runner.invoke(app, ["run", "error"])
+    assert result.exit_code == 1
+    assert "No command found in job: error." in result.stdout

{hydraflow-0.12.3 → hydraflow-0.12.5}/tests/core/io/test_iter_dirs.py

@@ -27,28 +27,44 @@ def setup(chdir):
         mlflow.log_text("5", "text.txt")
 
 
-def test_root_dir(root_dir: Path):
-    from hydraflow.core.io import get_root_dir
+def test_iter_experiment_dirs(root_dir: Path):
+    from hydraflow.core.io import get_experiment_name, iter_experiment_dirs
 
-    assert get_root_dir(root_dir) == root_dir
-    assert get_root_dir(root_dir.name) == root_dir
-    assert get_root_dir() == root_dir
+    names = [get_experiment_name(p) for p in iter_experiment_dirs(root_dir)]
+    assert sorted(names) == ["e1", "e2"]  # type: ignore
 
 
-def test_iter_experiment_dirs():
+@pytest.mark.parametrize(
+    ("e", "es"),
+    [("e1", ["e1"]), ("e*", ["e1", "e2"]), ("*", ["e1", "e2"]), ("*2", ["e2"])],
+)
+def test_iter_experiment_dirs_glob(root_dir: Path, e, es):
     from hydraflow.core.io import get_experiment_name, iter_experiment_dirs
 
-    names = [get_experiment_name(p) for p in iter_experiment_dirs()]
-    assert sorted(names) == ["e1", "e2"]  # type: ignore
+    names = [get_experiment_name(p) for p in iter_experiment_dirs(root_dir, e)]
+    assert sorted(names) == es  # type: ignore
 
 
-def test_iter_experiment_dirs_filter():
+def test_iter_experiment_dirs_filter(root_dir: Path):
     from hydraflow.core.io import get_experiment_name, iter_experiment_dirs
 
-    it = iter_experiment_dirs(experiment_names="e1")
+    it = iter_experiment_dirs(root_dir, experiment_names="e1")
     assert [get_experiment_name(p) for p in it] == ["e1"]
 
 
+def test_iter_experiment_dirs_filter_callable(root_dir: Path):
+    from hydraflow.core.io import get_experiment_name, iter_experiment_dirs
+
+    it = iter_experiment_dirs(root_dir, experiment_names=lambda name: name == "e2")
+    assert [get_experiment_name(p) for p in it] == ["e2"]
+
+
+def test_predicate_experiment_dir():
+    from hydraflow.core.io import predicate_experiment_dir
+
+    assert predicate_experiment_dir(Path()) is False
+
+
 def test_get_experiment_name_none(root_dir: Path):
     from hydraflow.core.io import get_experiment_name
 
@@ -62,21 +78,21 @@ def test_get_experiment_name_metafile_none(root_dir: Path):
     assert get_experiment_name(root_dir) is None
 
 
-def test_iter_run_dirs():
+def test_iter_run_dirs(root_dir: Path):
     from hydraflow.core.io import iter_run_dirs
 
-    assert len(list(iter_run_dirs())) == 5
+    assert len(list(iter_run_dirs(root_dir))) == 5
 
 
-def test_iter_artifacts_dirs():
+def test_iter_artifacts_dirs(root_dir: Path):
     from hydraflow.core.io import iter_artifacts_dirs
 
-    assert len(list(iter_artifacts_dirs())) == 5
+    assert len(list(iter_artifacts_dirs(root_dir))) == 5
 
 
-def test_iter_artifact_paths():
+def test_iter_artifact_paths(root_dir: Path):
     from hydraflow.core.io import iter_artifact_paths
 
-    it = iter_artifact_paths("text.txt")
+    it = iter_artifact_paths(root_dir, "text.txt")
     text = sorted("".join(p.read_text() for p in it))
     assert text == ["1", "2", "3", "4", "5"]

{hydraflow-0.12.3 → hydraflow-0.12.5}/tests/core/test_mlflow.py

@@ -81,10 +81,3 @@ def test_list_run_paths(experiment: Experiment):
 
     dirs = list_run_paths(experiment.name, "artifacts")
     assert all(d.is_dir() for d in dirs)
-
-
-@pytest.mark.parametrize("uri", [None, "test_mlflow"])
-def test_root_dir(experiment, uri):
-    from hydraflow.core.io import get_root_dir
-
-    assert get_root_dir(uri) == Path.cwd() / "test_mlflow"

{hydraflow-0.12.3 → hydraflow-0.12.5}/tests/executor/test_job.py

@@ -1,4 +1,3 @@
-import sys
 from pathlib import Path
 
 import pytest
@@ -60,66 +59,67 @@ def test_sweep_args(batches, i, x):
     assert batches[i][-3] == x
 
 
-def test_multirun_run(job: Job, tmp_path: Path):
-    from hydraflow.executor.job import multirun
+def test_iter_runs(job: Job, tmp_path: Path):
+    from hydraflow.executor.job import iter_batches, iter_runs
 
     path = tmp_path / "output.txt"
     file = Path(__file__).parent / "echo.py"
 
-    job.run = f"python {file.as_posix()} {path.as_posix()}"
-    multirun(job)
+    args = [file.as_posix(), path.as_posix()]
+    x = list(iter_runs("python", args, iter_batches(job)))
     assert path.read_text() == "b=5 a=1,2 b=6 a=1,2 c=7 a=3,4 c=8 a=3,4"
+    assert x[0].completed == 1
+    assert x[0].result.returncode == 0
+    assert x[1].completed == 2
+    assert x[1].result.returncode == 0
+    assert x[2].completed == 3
+    assert x[2].result.returncode == 0
 
 
-def test_multirun_run_error(job: Job):
-    from hydraflow.executor.job import multirun
+def test_iter_calls(job: Job, capsys: pytest.CaptureFixture):
+    from hydraflow.executor.job import iter_batches, iter_calls
 
-    job.run = "cmd /c exit 1" if sys.platform == "win32" else "false"
-    with pytest.raises(RuntimeError):
-        multirun(job)
-
-
-def test_multirun_call(job: Job, capsys: pytest.CaptureFixture):
-    from hydraflow.executor.job import multirun
-
-    job.call = "typer.echo"
-    multirun(job)
+    x = list(iter_calls("typer.echo", [], iter_batches(job)))
     out, _ = capsys.readouterr()
     assert "'b=5', 'a=1,2'" in out
     assert "'c=8', 'a=3,4'" in out
+    assert x[0].completed == 1
+    assert x[1].completed == 2
+    assert x[2].completed == 3
 
 
-def test_multirun_call_args(job: Job, capsys: pytest.CaptureFixture):
-    from hydraflow.executor.job import multirun
+def test_iter_calls_args(job: Job, capsys: pytest.CaptureFixture):
+    from hydraflow.executor.job import iter_batches, iter_calls
 
     job.call = "typer.echo a 'b c'"
-    multirun(job)
+    list(iter_calls("typer.echo", ["a", "b c"], iter_batches(job)))
     out, _ = capsys.readouterr()
     assert "['a', 'b c', '--multirun'," in out
 
 
-def test_multirun_call_error(job: Job):
-    from hydraflow.executor.job import multirun
+def test_submit(job: Job, capsys: pytest.CaptureFixture):
+    from hydraflow.executor.job import iter_batches, submit
 
-    job.call = "hydraflow.executor.job.multirun"
-    with pytest.raises(RuntimeError):
-        multirun(job)
+    submit("typer.echo", ["a"], iter_batches(job))
+    out, _ = capsys.readouterr()
+    assert out.startswith("[['a', '--multirun', 'b=5', 'a=1,2', 'hydra.job.name=test'")
+    assert "], ['a', '--multirun', 'b=6', 'a=1,2', 'hydra" in out
+    assert "], ['a', '--multirun', 'c=7', 'a=3,4', 'hydra" in out
+    assert "], ['a', '--multirun', 'c=8', 'a=3,4', 'hydra" in out
 
 
-def test_multirun_call_invalid(job: Job):
-    from hydraflow.executor.job import multirun
+def test_get_callable_error():
+    from hydraflow.executor.job import get_callable
 
-    job.call = "print"
     with pytest.raises(ValueError):
-        multirun(job)
+        get_callable("print")
 
 
-def test_multirun_call_not_found(job: Job):
-    from hydraflow.executor.job import multirun
+def test_get_callable_not_found():
+    from hydraflow.executor.job import get_callable
 
-    job.call = "hydraflow.invalid"
     with pytest.raises(ValueError):
-        multirun(job)
+        get_callable("hydraflow.invalid")
 
 
 def test_to_text(job: Job):