hydraflow 0.11.1__tar.gz → 0.12.1__tar.gz

{hydraflow-0.11.1 → hydraflow-0.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.11.1
+Version: 0.12.1
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.11.1 → hydraflow-0.12.1}/apps/quickstart.py

@@ -1,13 +1,16 @@
+from __future__ import annotations
+
 import logging
 from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
-import hydra
-import mlflow
 from hydra.core.config_store import ConfigStore
-from hydra.core.hydra_config import HydraConfig
 
 import hydraflow
 
+if TYPE_CHECKING:
+    from mlflow.entities import Run
+
 log = logging.getLogger(__name__)
 
 
@@ -21,13 +24,10 @@ cs = ConfigStore.instance()
 cs.store(name="config", node=Config)
 
 
-@hydra.main(config_name="config", version_base=None)
-def app(cfg: Config) -> None:
-    hc = HydraConfig.get()
-    mlflow.set_experiment(hc.job.name)
-
-    with hydraflow.start_run(cfg):
-        log.info(f"{cfg.width=}, {cfg.height=}")
+@hydraflow.main(Config)
+def app(run: Run, cfg: Config) -> None:
+    log.info(run.info.run_id)
+    log.info(cfg)
 
 
 if __name__ == "__main__":

{hydraflow-0.11.1 → hydraflow-0.12.1}/docs/usage/quickstart.md

@@ -2,24 +2,22 @@
 
 ## Hydra application
 
-The following example demonstrates how to use Hydraflow with a Hydra application.
-There are two main steps to using Hydraflow:
+The following example demonstrates how to use a Hydraflow application.
 
-1. Set the MLflow experiment using the Hydra job name.
-2. Start a new MLflow run that logs the Hydra configuration.
-
-```python title="apps/quickstart.py" linenums="1" hl_lines="24 26"
+```python title="apps/quickstart.py" linenums="1"
 --8<-- "apps/quickstart.py"
 ```
 
-### Start a new MLflow run
+### Hydraflow's `main` decorator
 
-[`hydraflow.start_run`][] starts a new MLflow run that logs the Hydra configuration.
-It returns the started run so that it can be used to log metrics, parameters, and artifacts
-within the context of the run.
+[`hydraflow.main`][] starts a new MLflow run that logs the Hydra configuration.
+The decorated function must have two arguments: `Run` and `Config`.
+The `Run` argument is the current MLflow run.
+The `Config` argument is the Hydra configuration.
 
 ```python
-with hydraflow.start_run(cfg) as run:
+@hydraflow.main(Config)
+def app(run: Run, cfg: Config) -> None:
     pass
 ```
 
@@ -31,7 +29,7 @@ rm -rf mlruns outputs multirun
 
 ### Single-run
 
-Run the Hydra application as a normal Python script.
+Run the Hydraflow application as a normal Python script.
 
 ```console exec="1" source="console"
 $ python apps/quickstart.py
@@ -43,8 +41,12 @@ Check the MLflow CLI to view the experiment.
 $ mlflow experiments search
 ```
 
+The experiment name is the name of the Hydra job.
+
 ### Multi-run
 
+Run the Hydraflow application with multiple configurations.
+
 ```console exec="1" source="console"
 $ python apps/quickstart.py -m width=400,600 height=100,200,300
 ```
@@ -53,6 +55,8 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 
 ### Run collection
 
+The `RunCollection` object is a collection of runs.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> import hydraflow
 >>> rc = hydraflow.list_runs("quickstart")
@@ -61,35 +65,47 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 
 ### Retrieve a run
 
+The `RunCollection` object has a `first` and `last` method that
+returns the first and last run in the collection.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> run = rc.first()
 >>> print(type(run))
 ```
 
 ```pycon exec="1" source="console" session="quickstart"
+>>> run = rc.last()
 >>> cfg = hydraflow.load_config(run)
->>> print(type(cfg))
 >>> print(cfg)
 ```
 
+The `load_config` function loads the Hydra configuration from the run.
+
 ```pycon exec="1" source="console" session="quickstart"
->>> run = rc.last()
 >>> cfg = hydraflow.load_config(run)
+>>> print(type(cfg))
 >>> print(cfg)
 ```
 
 ### Filter runs
 
+The `filter` method filters the runs by the given key-value pairs.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> filtered = rc.filter(width=400)
 >>> print(filtered)
 ```
 
+If the value is a list, the run will be included if the value is in the list.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> filtered = rc.filter(height=[100, 300])
 >>> print(filtered)
 ```
 
+If the value is a tuple, the run will be included if the value is between the tuple.
+The start and end of the tuple are inclusive.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> filtered = rc.filter(height=(100, 300))
 >>> print(filtered)
@@ -97,12 +113,16 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 
 ### Group runs
 
+The `groupby` method groups the runs by the given key.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> grouped = rc.groupby("width")
 >>> for key, group in grouped.items():
 ...     print(key, group)
 ```
 
+The `groupby` method can also take a list of keys.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> grouped = rc.groupby(["height"])
 >>> for key, group in grouped.items():
@@ -111,6 +131,13 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 
 ### Config dataframe
 
+The `data.config` attribute returns a pandas DataFrame
+of the Hydra configuration.
+
 ```pycon exec="1" source="console" session="quickstart"
 >>> print(rc.data.config)
 ```
+
+```bash exec="on"
+rm -rf mlruns outputs multirun
+```

{hydraflow-0.11.1 → hydraflow-0.12.1}/mkdocs.yaml

@@ -53,4 +53,4 @@ nav:
   - Home: index.md
   - Usage:
       - usage/quickstart.md
-  - Reference: $api/hydraflow.**
+  - Reference: $api/hydraflow.***

{hydraflow-0.11.1 → hydraflow-0.12.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.11.1"
+version = "0.12.1"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -96,8 +96,14 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"apps/*.py" = ["D", "G", "INP"]
+"apps/*.py" = ["D", "G", "INP", "T"]
 "src/hydraflow/cli.py" = ["ANN", "D"]
 "src/hydraflow/core/main.py" = ["ANN201", "D401"]
 "src/hydraflow/executor/conf.py" = ["ANN", "D"]
 "tests/*" = ["A001", "ANN", "ARG", "D", "FBT", "PD", "PLR", "PT", "S", "SLF"]
+
+[tool.pyright]
+include = ["src", "tests"]
+strictDictionaryInference = true
+strictListInference = true
+strictSetInference = true

{hydraflow-0.11.1 → hydraflow-0.12.1}/src/hydraflow/core/main.py

@@ -16,6 +16,7 @@ used to wrap experiment entry points. This decorator handles:
 
 Example:
     ```python
+    import hydraflow
     from dataclasses import dataclass
     from mlflow.entities import Run
 
@@ -24,7 +25,7 @@ Example:
         learning_rate: float
         batch_size: int
 
-    @main(Config)
+    @hydraflow.main(Config)
     def train(run: Run, config: Config):
         # Your training code here
         pass

{hydraflow-0.11.1 → hydraflow-0.12.1}/src/hydraflow/executor/conf.py

@@ -7,7 +7,7 @@ from dataclasses import dataclass, field
 class Step:
     batch: str = ""
     args: str = ""
-    configs: str = ""
+    with_: str = ""
 
 
 @dataclass
@@ -15,7 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
-    configs: str = ""
+    with_: str = ""
     steps: list[Step] = field(default_factory=list)
 
 

{hydraflow-0.11.1 → hydraflow-0.12.1}/src/hydraflow/executor/io.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from omegaconf import OmegaConf
+from omegaconf import DictConfig, ListConfig, OmegaConf
 
 from .conf import HydraflowConf
 
@@ -35,7 +35,26 @@ def load_config() -> HydraflowConf:
 
     cfg = OmegaConf.load(path)
 
-    return OmegaConf.merge(schema, cfg)  # type: ignore
+    if not isinstance(cfg, DictConfig):
+        return schema
+
+    rename_with(cfg)
+
+    return OmegaConf.merge(schema, cfg)  # type: ignore[return-value]
+
+
+def rename_with(cfg: DictConfig) -> None:
+    """Rename the `with` field to `with_`."""
+    if "with" in cfg:
+        cfg["with_"] = cfg.pop("with")
+
+    for key in list(cfg.keys()):
+        if isinstance(cfg[key], DictConfig):
+            rename_with(cfg[key])
+        elif isinstance(cfg[key], ListConfig):
+            for item in cfg[key]:
+                if isinstance(item, DictConfig):
+                    rename_with(item)
 
 
 def get_job(name: str) -> Job:

{hydraflow-0.11.1 → hydraflow-0.12.1}/src/hydraflow/executor/job.py

@@ -69,10 +69,10 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
     """
     job_name = f"hydra.job.name={job.name}"
-    job_configs = shlex.split(job.configs)
+    job_configs = shlex.split(job.with_)
 
     for step in job.steps:
-        configs = shlex.split(step.configs) or job_configs
+        configs = shlex.split(step.with_) or job_configs
 
         for args in iter_args(step.batch, step.args):
             sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ULID()}"

{hydraflow-0.11.1 → hydraflow-0.12.1}/src/hydraflow/executor/parser.py

@@ -83,7 +83,7 @@ def count_decimal_places(x: str) -> int:
 
 
 def is_number(x: str) -> bool:
-    """Check if a string is a number.
+    """Check if a string represents a valid number.
 
     Args:
         x (str): The string to check.
@@ -140,18 +140,54 @@ def _get_range(arg: str) -> tuple[float, float, float]:
 
 
 def _arange(start: float, step: float, stop: float) -> list[float]:
+    """Generate a range of floating point numbers.
+
+    This function generates a range of floating point numbers
+    with protection against rounding errors.
+
+    Args:
+        start (float): The starting value.
+        step (float): The step size.
+        stop (float): The end value (inclusive).
+
+    Returns:
+        list[float]: A list of floating point numbers from start to stop
+        (inclusive) with the given step.
+
+    """
+    if step == 0:
+        raise ValueError("Step cannot be zero")
+
+    epsilon = 1e-10
+
+    decimal_places = max(
+        count_decimal_places(str(start)),
+        count_decimal_places(str(step)),
+        count_decimal_places(str(stop)),
+    )
+
     result = []
     current = start
 
-    while current <= stop if step > 0 else current >= stop:
-        result.append(current)
-        current += step
+    if step > 0:
+        while current <= stop + epsilon:
+            rounded = round(current, decimal_places)
+            result.append(rounded)
+            current += step
+    else:
+        while current >= stop - epsilon:
+            rounded = round(current, decimal_places)
+            result.append(rounded)
+            current += step
 
     return result
 
 
 def split_suffix(arg: str) -> tuple[str, str]:
-    """Split a string into prefix and suffix.
+    """Split a string into the prefix and suffix.
+
+    The suffix is the part of the string that starts with a colon (:).
+    The prefix is the part of the string that precedes the suffix.
 
     Args:
         arg (str): The string to split.
@@ -194,6 +230,16 @@ def add_exponent(value: str, exponent: str) -> str:
     Returns:
         str: The value with the exponent added.
 
+    Examples:
+        >>> add_exponent("1", "e3")
+        '1e3'
+        >>> add_exponent("1", "")
+        '1'
+        >>> add_exponent("0", "e-3")
+        '0'
+        >>> add_exponent("0.0", "e-3")
+        '0.0'
+
     """
     if value in ["0", "0.", "0.0"] or not exponent:
         return value
@@ -201,43 +247,6 @@ def add_exponent(value: str, exponent: str) -> str:
     return f"{value}{exponent}"
 
 
-def collect_values(arg: str) -> list[str]:
-    """Collect a list of values from a range argument.
-
-    Collect all individual values within a numeric range
-    represented by a string (e.g., `1:4`) and return them
-    as a list of strings.
-    Support both integer and floating-point ranges.
-
-    Args:
-        arg (str): The argument to collect.
-
-    Returns:
-        list[str]: A list of the collected values.
-
-    """
-    if "(" in arg:
-        return collect_parentheses(arg)
-
-    if ":" not in arg:
-        return [arg]
-
-    arg, exponent = split_suffix(arg)
-
-    if ":" not in arg:
-        return [f"{arg}{exponent}"]
-
-    rng = _get_range(arg)
-
-    if all(isinstance(x, int) for x in rng):
-        values = [str(x) for x in _arange(*rng)]
-    else:
-        n = max(*(count_decimal_places(x) for x in arg.split(":")))
-        values = [str(round(x, n)) for x in _arange(*rng)]
-
-    return [add_exponent(x, exponent) for x in values]
-
-
 def split_parentheses(arg: str) -> Iterator[str]:
     """Split a string with parentheses into a list of strings.
 
@@ -290,8 +299,59 @@ def collect_parentheses(arg: str) -> list[str]:
     return ["".join(x[::-1]) for x in product(*it[::-1])]
 
 
+def collect_values(arg: str) -> list[str]:
+    """Collect a list of values from a range argument.
+
+    Collect all individual values within a numeric range
+    represented by a string (e.g., `1:4`) and return them
+    as a list of strings.
+    Support both integer and floating-point ranges.
+
+    Args:
+        arg (str): The argument to collect.
+
+    Returns:
+        list[str]: A list of the collected values.
+
+    Examples:
+        >>> collect_values("1:4")
+        ['1', '2', '3', '4']
+        >>> collect_values("1.2:0.1:1.4:k")
+        ['1.2e3', '1.3e3', '1.4e3']
+        >>> collect_values("0.1")
+        ['0.1']
+        >>> collect_values("4:M")
+        ['4e6']
+        >>> collect_values("(1:3,5:7)M")
+        ['1e6', '2e6', '3e6', '5e6', '6e6', '7e6']
+        >>> collect_values("(1,5)e-(1:3)")
+        ['1e-1', '5e-1', '1e-2', '5e-2', '1e-3', '5e-3']
+
+    """
+    if "(" in arg:
+        return collect_parentheses(arg)
+
+    if ":" not in arg:
+        return [arg]
+
+    arg, exponent = split_suffix(arg)
+
+    if ":" not in arg:
+        return [f"{arg}{exponent}"]
+
+    rng = _get_range(arg)
+
+    if all(isinstance(x, int) for x in rng):
+        values = [str(x) for x in _arange(*rng)]
+    else:
+        n = max(*(count_decimal_places(x) for x in arg.split(":")))
+        values = [str(round(x, n)) for x in _arange(*rng)]
+
+    return [add_exponent(x, exponent) for x in values]
+
+
 def split(arg: str) -> list[str]:
-    r"""Split a string by top-level commas.
+    """Split a string by top-level commas.
 
     Splits a string by commas while respecting nested structures.
     Commas inside brackets and quotes are ignored, only splitting
@@ -389,7 +449,7 @@ def split_arg(arg: str) -> tuple[str, str, str]:
     key, value = arg.split("=")
 
     if "/" in key:
-        key, suffix = key.split("/", 1)
+        key, suffix = key.split("/")
         return key, suffix, value
 
     return key, "", value

{hydraflow-0.11.1 → hydraflow-0.12.1}/tests/cli/hydraflow.yaml

@@ -13,10 +13,10 @@ jobs:
         args: count=100
   parallel:
     run: python app.py
-    configs: hydra/launcher=joblib hydra.launcher.n_jobs=2
+    with: hydra/launcher=joblib hydra.launcher.n_jobs=2
     steps:
       - batch: name=a
         args: count=1:4
       - batch: name=b
         args: count=11:14
-        configs: hydra/launcher=joblib hydra.launcher.n_jobs=4
+        with: hydra/launcher=joblib hydra.launcher.n_jobs=4

{hydraflow-0.11.1 → hydraflow-0.12.1}/tests/executor/test_conf.py

@@ -25,10 +25,11 @@ def test_none():
 
 
 def test_job(config):
-    cfg = config("jobs:\n  a:\n    run: a.test\n")
+    cfg = config("jobs:\n  a:\n    run: a.test\n    with: --opt1 --opt2\n")
     assert cfg.jobs["a"].run == "a.test"
+    assert cfg.jobs["a"].with_ == "--opt1 --opt2"
 
 
 def test_step(config):
-    cfg = config("jobs:\n  a:\n    steps:\n      - configs: --opt1 --opt2\n")
-    assert cfg.jobs["a"].steps[0].configs == "--opt1 --opt2"
+    cfg = config("jobs:\n  a:\n    steps:\n      - with: --opt1 --opt2\n")
+    assert cfg.jobs["a"].steps[0].with_ == "--opt1 --opt2"

{hydraflow-0.11.1 → hydraflow-0.12.1}/tests/executor/test_io.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+from omegaconf import DictConfig
 
 
 @pytest.mark.parametrize("file", ["hydraflow.yaml", "hydraflow.yml"])
@@ -16,3 +17,15 @@ def test_find_config_none(chdir):
     from hydraflow.executor.io import find_config_file
 
     assert find_config_file() is None
+
+
+def test_load_config_list(chdir):
+    from hydraflow.executor.io import load_config
+
+    Path("hydraflow.yaml").write_text("- a\n- b\n")
+
+    cfg = load_config()
+    assert isinstance(cfg, DictConfig)
+    assert cfg.jobs == {}
+
+    Path("hydraflow.yaml").unlink()

{hydraflow-0.11.1 → hydraflow-0.12.1}/tests/executor/test_parser.py

@@ -24,6 +24,7 @@ def test_count_decimal_places(s, x):
         ("1:2", (1, 1, 2)),
         (":2", (0, 1, 2)),
         ("0.1:0.1:0.4", (0.1, 0.1, 0.4)),
+        ("1.2:0.1:1.4", (1.2, 0.1, 1.4)),
     ],
 )
 def test_get_range(s, x):
@@ -49,6 +50,28 @@ def test_get_range_errors(arg, expected_exception, expected_message):
     assert str(excinfo.value) == expected_message
 
 
+@pytest.mark.parametrize(
+    ("start", "step", "stop", "expected"),
+    [
+        (1.0, 2.0, 5.0, [1.0, 3.0, 5.0]),
+        (1.2, 0.1, 1.4, [1.2, 1.3, 1.4]),
+        (1.4, -0.1, 1.2, [1.4, 1.3, 1.2]),
+        (1.02e-3, 0.01e-3, 1.04e-3, [0.00102, 0.00103, 0.00104]),
+    ],
+)
+def test_arange(start, step, stop, expected):
+    from hydraflow.executor.parser import _arange
+
+    assert _arange(start, step, stop) == expected
+
+
+def test_arange_error():
+    from hydraflow.executor.parser import _arange
+
+    with pytest.raises(ValueError):
+        _arange(1.0, 0.0, 1.0)
+
+
 @pytest.mark.parametrize(
     ("s", "x"),
     [
@@ -100,6 +123,7 @@ def test_split_suffix(s, x):
         ("4.5:-1.5:-4.5", ["4.5", "3.0", "1.5", "0.0", "-1.5", "-3.0", "-4.5"]),
         ("1:2:u", ["1e-6", "2e-6"]),
         ("1:.25:2:n", ["1e-9", "1.25e-9", "1.5e-9", "1.75e-9", "2.0e-9"]),
+        ("1.2:0.1:1.4:k", ["1.2e3", "1.3e3", "1.4e3"]),
         ("1:2:e2", ["1e2", "2e2"]),
         (":2:e2", ["0", "1e2", "2e2"]),
         ("-2:2:k", ["-2e3", "-1e3", "0", "1e3", "2e3"]),