hydraflow 0.10.1__py3-none-any.whl → 0.11.0__py3-none-any.whl

hydraflow/cli.py

@@ -2,54 +2,53 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Annotated
+from typing import Annotated
 
 import typer
 from rich.console import Console
 from typer import Argument, Option
 
-from hydraflow.executor.io import load_config
-
-if TYPE_CHECKING:
-    from hydraflow.executor.job import Job
+from hydraflow.executor.io import get_job, load_config
 
 app = typer.Typer(add_completion=False)
 console = Console()
 
 
-def get_job(name: str) -> Job:
-    cfg = load_config()
-    job = cfg.jobs[name]
-
-    if not job.name:
-        job.name = name
-
-    return job
-
-
 @app.command()
 def run(
     name: Annotated[str, Argument(help="Job name.", show_default=False)],
+    *,
+    dry_run: Annotated[
+        bool,
+        Option("--dry-run", help="Perform a dry run"),
+    ] = False,
 ) -> None:
     """Run a job."""
     import mlflow
 
-    from hydraflow.executor.job import multirun
+    from hydraflow.executor.job import multirun, to_text
 
     job = get_job(name)
-    mlflow.set_experiment(job.name)
-    multirun(job)
+    if dry_run:
+        typer.echo(to_text(job))
+    else:
+        mlflow.set_experiment(job.name)
+        multirun(job)
 
 
 @app.command()
 def show(
-    name: Annotated[str, Argument(help="Job name.", show_default=False)],
+    name: Annotated[str, Argument(help="Job name.", show_default=False)] = "",
 ) -> None:
-    """Show a job."""
-    from hydraflow.executor.job import show
+    """Show the hydraflow config."""
+    from omegaconf import OmegaConf
 
-    job = get_job(name)
-    show(job)
+    if name:
+        cfg = get_job(name)
+    else:
+        cfg = load_config()
+
+    typer.echo(OmegaConf.to_yaml(cfg))
 
 
 @app.callback(invoke_without_command=True)

hydraflow/executor/conf.py

@@ -5,9 +5,9 @@ from dataclasses import dataclass, field
 
 @dataclass
 class Step:
-    args: str = ""
     batch: str = ""
-    options: str = ""
+    args: str = ""
+    configs: str = ""
 
 
 @dataclass
@@ -15,6 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
+    configs: str = ""
     steps: list[Step] = field(default_factory=list)
 
 

hydraflow/executor/io.py

@@ -3,11 +3,15 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from omegaconf import OmegaConf
 
 from .conf import HydraflowConf
 
+if TYPE_CHECKING:
+    from .job import Job
+
 
 def find_config_file() -> Path | None:
     """Find the hydraflow config file."""
@@ -32,3 +36,14 @@ def load_config() -> HydraflowConf:
     cfg = OmegaConf.load(path)
 
     return OmegaConf.merge(schema, cfg)  # type: ignore
+
+
+def get_job(name: str) -> Job:
+    """Get a job from the config."""
+    cfg = load_config()
+    job = cfg.jobs[name]
+
+    if not job.name:
+        job.name = name
+
+    return job

hydraflow/executor/job.py

@@ -12,7 +12,7 @@ The module supports two execution modes:
 2. Python function calls
 
 Each job can consist of multiple steps, and each step can have its own
-arguments and options that will be expanded into multiple runs.
+arguments and configurations that will be expanded into multiple runs.
 """
 
 from __future__ import annotations
@@ -20,6 +20,7 @@ from __future__ import annotations
 import importlib
 import shlex
 import subprocess
+import sys
 from subprocess import CalledProcessError
 from typing import TYPE_CHECKING
 
@@ -30,27 +31,27 @@ from .parser import collect, expand
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from .conf import Job, Step
+    from .conf import Job
 
 
-def iter_args(step: Step) -> Iterator[list[str]]:
+def iter_args(batch: str, args: str) -> Iterator[list[str]]:
     """Iterate over combinations generated from parsed arguments.
 
     Generate all possible combinations of arguments by parsing and
     expanding each one, yielding them as an iterator.
 
     Args:
-        step (Step): The step to parse.
+        batch (str): The batch to parse.
+        args (str): The arguments to parse.
 
     Yields:
         list[str]: a list of the parsed argument combinations.
 
     """
-    args = collect(step.args)
-    options = [o for o in step.options.split(" ") if o]
+    args_ = collect(args)
 
-    for batch in expand(step.batch):
-        yield [*options, *sorted([*batch, *args])]
+    for batch_ in expand(batch):
+        yield [*batch_, *args_]
 
 
 def iter_batches(job: Job) -> Iterator[list[str]]:
@@ -68,11 +69,14 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
     """
     job_name = f"hydra.job.name={job.name}"
+    job_configs = shlex.split(job.configs)
 
     for step in job.steps:
-        for args in iter_args(step):
+        configs = shlex.split(step.configs) or job_configs
+
+        for args in iter_args(step.batch, step.args):
             sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ULID()}"
-            yield ["--multirun", sweep_dir, job_name, *args]
+            yield ["--multirun", *args, job_name, sweep_dir, *configs]
 
 
 def multirun(job: Job) -> None:
@@ -99,6 +103,8 @@ def multirun(job: Job) -> None:
 
     if job.run:
         base_cmds = shlex.split(job.run)
+        if base_cmds[0] == "python" and sys.platform == "win32":
+            base_cmds[0] = sys.executable
 
         for args in it:
             cmds = [*base_cmds, *args]
@@ -132,24 +138,31 @@ def multirun(job: Job) -> None:
                 raise RuntimeError(msg) from e
 
 
-def show(job: Job) -> None:
-    """Show the job configuration.
+def to_text(job: Job) -> str:
+    """Convert the job configuration to a string.
 
-    This function shows the job configuration for a given job.
+    This function returns the job configuration for a given job.
 
     Args:
         job (Job): The job configuration to show.
 
+    Returns:
+        str: The job configuration.
+
     """
+    text = ""
+
     it = iter_batches(job)
 
     if job.run:
         base_cmds = shlex.split(job.run)
         for args in it:
             cmds = " ".join([*base_cmds, *args])
-            print(cmds)  # noqa: T201
+            text += f"{cmds}\n"
 
     elif job.call:
-        print(f"call: {job.call}")  # noqa: T201
+        text = f"call: {job.call}\n"
         for args in it:
-            print(f"args: {args}")  # noqa: T201
+            text += f"args: {args}\n"
+
+    return text.rstrip()

{hydraflow-0.10.1.dist-info → hydraflow-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.10.1
+Version: 0.11.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.10.1.dist-info → hydraflow-0.11.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 hydraflow/__init__.py,sha256=f2KO2iF7um-nNmayNyEr7TWG4UICOXy7YAN1d3qu0OY,936
-hydraflow/cli.py,sha256=gbDPj49azP8CCGxkxU0rksh1-gCyjP0VkVYH34ktcsA,1338
+hydraflow/cli.py,sha256=b-M368amGUblOOqOi7JuBsnzLuaNqgpTit9ZRj0qqac,1410
 hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
@@ -13,12 +13,12 @@ hydraflow/entities/run_collection.py,sha256=E8IRBgxCnJE_IPCaSmS2mc9GtDXXLBfc7GHv
 hydraflow/entities/run_data.py,sha256=Y2_Lc-BdQ7nXhcEIjdHGHIkLrXsmAktOftESEwYOY8o,1602
 hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/executor/conf.py,sha256=q_FrPXQJCVGKS1FYnGRGqTUgMQeMBkaVPW2mtQc8oxk,384
-hydraflow/executor/io.py,sha256=4nafwge6vHanYFuEHxd0LRv_3ZLgMpV50qSbssZNe3Q,696
-hydraflow/executor/job.py,sha256=_f_7lfPsPMGBNB7ZW50-MCZX-BQi-s0_Ajg3DjqHx5s,4553
+hydraflow/executor/conf.py,sha256=SJNiQ87MXMlpDfdm0POcv55MY3GS5FUh5wT7u3XU3oU,406
+hydraflow/executor/io.py,sha256=xV3m-nV9eKbu9Fb7u04J2bfmR_Ky3jTEJjq4QC2m6V4,954
+hydraflow/executor/job.py,sha256=yiVbAYgsZZtSTRER-H5pUopULeygzPNzKlSkl27yFI4,4856
 hydraflow/executor/parser.py,sha256=MO8VU0uVQZeku6kbw8Urid_5QEcnR8atd5h-yDP5OhQ,10147
-hydraflow-0.10.1.dist-info/METADATA,sha256=1fBBNz3-3gczIXYs9tq7lvzDySjz1BlFMQsO73KgCrg,4574
-hydraflow-0.10.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.10.1.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.10.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.10.1.dist-info/RECORD,,
+hydraflow-0.11.0.dist-info/METADATA,sha256=MIDZ7Vec2SDAcj0S1SvghtlN5RQTpfR1N3MLlwoLD9A,4574
+hydraflow-0.11.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.11.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.11.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.11.0.dist-info/RECORD,,