hydraflow 0.15.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

hydraflow/core/run.py

@@ -249,29 +249,40 @@ class Run[C, I = None]:
             if force or OmegaConf.select(cfg, k, default=MISSING) is MISSING:
                 OmegaConf.update(cfg, k, v, force_add=True)
 
-    def get(self, key: str) -> Any:
+    def get(self, key: str, default: Any = MISSING) -> Any:
         """Get a value from the information or configuration.
 
         Args:
-            key: The key to look for. Can use dot notation for nested keys
-                in configuration.
+            key: The key to look for. Can use dot notation for
+                nested keys in configuration.
+            default: Value to return if the key is not found.
+                If not provided, AttributeError will be raised.
 
         Returns:
-            Any: The value associated with the key.
+            Any: The value associated with the key, or the
+            default value if the key is not found and a default
+            is provided.
 
         Raises:
-            AttributeError: If the key is not found in any of the components.
+            AttributeError: If the key is not found and
+                no default is provided.
 
         """
         value = OmegaConf.select(self.cfg, key, default=MISSING)  # type: ignore
         if value is not MISSING:
             return value
 
+        if self.impl and hasattr(self.impl, key):
+            return getattr(self.impl, key)
+
         info = self.info.to_dict()
         if key in info:
             return info[key]
 
-        msg = f"Key not found: {key}"
+        if default is not MISSING:
+            return default
+
+        msg = f"No such key: {key}"
         raise AttributeError(msg)
 
     def predicate(self, key: str, value: Any) -> bool:
@@ -298,32 +309,35 @@ class Run[C, I = None]:
 
         """
         attr = self.get(key)
+        return _predicate(attr, value)
 
-        if callable(value):
-            return bool(value(attr))
+    def to_dict(self) -> dict[str, Any]:
+        """Convert the Run to a dictionary."""
+        info = self.info.to_dict()
+        cfg = OmegaConf.to_container(self.cfg)
+        return info | _flatten_dict(cfg)  # type: ignore
 
-        if isinstance(value, ListConfig):
-            value = list(value)
 
-        if isinstance(value, list | set) and not _is_iterable(attr):
-            return attr in value
+def _predicate(attr: Any, value: Any) -> bool:
+    if callable(value):
+        return bool(value(attr))
 
-        if isinstance(value, tuple) and len(value) == 2 and not _is_iterable(attr):
-            return value[0] <= attr <= value[1]
+    if isinstance(value, ListConfig):
+        value = list(value)
 
-        if _is_iterable(value):
-            value = list(value)
+    if isinstance(value, list | set) and not _is_iterable(attr):
+        return attr in value
 
-        if _is_iterable(attr):
-            attr = list(attr)
+    if isinstance(value, tuple) and len(value) == 2 and not _is_iterable(attr):
+        return value[0] <= attr <= value[1]
 
-        return attr == value
+    if _is_iterable(value):
+        value = list(value)
 
-    def to_dict(self) -> dict[str, Any]:
-        """Convert the Run to a dictionary."""
-        info = self.info.to_dict()
-        cfg = OmegaConf.to_container(self.cfg)
-        return info | _flatten_dict(cfg)  # type: ignore
+    if _is_iterable(attr):
+        attr = list(attr)
+
+    return attr == value
 
 
 def _is_iterable(value: Any) -> bool:

hydraflow/executor/conf.py

@@ -4,10 +4,10 @@ from dataclasses import dataclass, field
 
 
 @dataclass
-class Step:
-    batch: str = ""
-    args: str = ""
-    with_: str = ""
+class Set:
+    each: str = ""
+    all: str = ""
+    add: str = ""
 
 
 @dataclass
@@ -16,8 +16,8 @@ class Job:
     run: str = ""
     call: str = ""
     submit: str = ""
-    with_: str = ""
-    steps: list[Step] = field(default_factory=list)
+    add: str = ""
+    sets: list[Set] = field(default_factory=list)
 
 
 @dataclass

hydraflow/executor/io.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import TYPE_CHECKING
 
-from omegaconf import DictConfig, ListConfig, OmegaConf
+from omegaconf import DictConfig, OmegaConf
 
 from .conf import HydraflowConf
 
@@ -38,25 +38,9 @@ def load_config() -> HydraflowConf:
     if not isinstance(cfg, DictConfig):
         return schema
 
-    rename_with(cfg)
-
     return OmegaConf.merge(schema, cfg)  # type: ignore[return-value]
 
 
-def rename_with(cfg: DictConfig) -> None:
-    """Rename the `with` field to `with_`."""
-    if "with" in cfg:
-        cfg["with_"] = cfg.pop("with")
-
-    for key in list(cfg.keys()):
-        if isinstance(cfg[key], DictConfig):
-            rename_with(cfg[key])
-        elif isinstance(cfg[key], ListConfig):
-            for item in cfg[key]:
-                if isinstance(item, DictConfig):
-                    rename_with(item)
-
-
 def get_job(name: str) -> Job:
     """Get a job from the config."""
     cfg = load_config()

hydraflow/executor/job.py

@@ -2,7 +2,7 @@
 
 This module provides functionality for executing jobs in HydraFlow, including:
 
-- Argument parsing and expansion for job steps
+- Argument parsing and expansion for job parameter sets
 - Batch processing of Hydra configurations
 - Execution of jobs via shell commands or Python functions
 
@@ -11,8 +11,9 @@ The module supports two execution modes:
 1. Shell command execution
 2. Python function calls
 
-Each job can consist of multiple steps, and each step can have its own
-arguments and configurations that will be expanded into multiple runs.
+Each job can consist of multiple parameter sets, and each parameter
+set can have its own arguments and configurations that will be expanded
+into multiple runs.
 """
 
 from __future__ import annotations
@@ -39,24 +40,24 @@ if TYPE_CHECKING:
     from .conf import Job
 
 
-def iter_args(batch: str, args: str) -> Iterator[list[str]]:
+def iter_args(each: str, all_: str) -> Iterator[list[str]]:
     """Iterate over combinations generated from parsed arguments.
 
     Generate all possible combinations of arguments by parsing and
     expanding each one, yielding them as an iterator.
 
     Args:
-        batch (str): The batch to parse.
-        args (str): The arguments to parse.
+        each (str): The 'each' parameter to parse.
+        all_ (str): The 'all' parameter to parse.
 
     Yields:
         list[str]: a list of the parsed argument combinations.
 
     """
-    args_ = collect(args)
+    all_params = collect(all_)
 
-    for batch_ in expand(batch):
-        yield [*batch_, *args_]
+    for each_params in expand(each):
+        yield [*each_params, *all_params]
 
 
 def iter_batches(job: Job) -> Iterator[list[str]]:
@@ -74,14 +75,40 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
     """
     job_name = f"hydra.job.name={job.name}"
-    job_configs = shlex.split(job.with_)
+    job_add = shlex.split(job.add)
 
-    for step in job.steps:
-        configs = shlex.split(step.with_) or job_configs
+    for set_ in job.sets:
+        add = merge_args(job_add, shlex.split(set_.add)) if set_.add else job_add
 
-        for args in iter_args(step.batch, step.args):
+        for args in iter_args(set_.each, set_.all):
             sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ULID()}"
-            yield ["--multirun", *args, job_name, sweep_dir, *configs]
+            yield ["--multirun", *args, job_name, sweep_dir, *add]
+
+
+def merge_args(first: list[str], second: list[str]) -> list[str]:
+    """Merge two lists of arguments.
+
+    This function merges two lists of arguments by checking for conflicts
+    and resolving them by keeping the values from the second list.
+
+    Args:
+        first (list[str]): The first list of arguments.
+        second (list[str]): The second list of arguments.
+
+    Returns:
+        list[str]: A merged list of arguments.
+
+    """
+    merged = {}
+
+    for item in [*first, *second]:
+        if "=" in item:
+            key, value = item.split("=", 1)
+            merged[key] = value
+        else:
+            merged[item] = None
+
+    return [k if v is None else f"{k}={v}" for k, v in merged.items()]
 
 
 @dataclass

hydraflow/executor/parser.py

@@ -165,25 +165,26 @@ SUFFIX_EXPONENT = {
 
 
 def _get_range(arg: str) -> tuple[float, float, float]:
+    """Return a tuple of (start, stop, step)."""
     args = [to_number(x) for x in arg.split(":")]
 
     if len(args) == 2:
         if args[0] > args[1]:
             raise ValueError("start cannot be greater than stop")
 
-        return (args[0], 1, args[1])
+        return (args[0], args[1], 1)
 
-    if args[1] == 0:
+    if args[2] == 0:
         raise ValueError("step cannot be zero")
-    if args[1] > 0 and args[0] > args[2]:
+    if args[2] > 0 and args[0] > args[1]:
         raise ValueError("start cannot be greater than stop")
-    if args[1] < 0 and args[0] < args[2]:
+    if args[2] < 0 and args[0] < args[1]:
         raise ValueError("start cannot be less than stop")
 
     return args[0], args[1], args[2]
 
 
-def _arange(start: float, step: float, stop: float) -> list[float]:
+def _arange(start: float, stop: float, step: float) -> list[float]:
     """Generate a range of floating point numbers.
 
     This function generates a range of floating point numbers
@@ -191,8 +192,8 @@ def _arange(start: float, step: float, stop: float) -> list[float]:
 
     Args:
         start (float): The starting value.
-        step (float): The step size.
         stop (float): The end value (inclusive).
+        step (float): The step size.
 
     Returns:
         list[float]: A list of floating point numbers from start to stop
@@ -323,7 +324,7 @@ def collect_parentheses(arg: str) -> list[str]:
         list[str]: A list of the collected values.
 
     Examples:
-        >>> collect_parentheses("(1:3,5:2:9,20)k")
+        >>> collect_parentheses("(1:3,5:9:2,20)k")
         ['1e3', '2e3', '3e3', '5e3', '7e3', '9e3', '20e3']
         >>> collect_parentheses("2e(-1,-2,-3)")
         ['2e-1', '2e-2', '2e-3']
@@ -352,7 +353,7 @@ def collect_values(arg: str) -> list[str]:
     Examples:
         >>> collect_values("1:4")
         ['1', '2', '3', '4']
-        >>> collect_values("1.2:0.1:1.4:k")
+        >>> collect_values("1.2:1.4:0.1:k")
         ['1.2e3', '1.3e3', '1.4e3']
         >>> collect_values("0.1")
         ['0.1']

{hydraflow-0.15.0.dist-info → hydraflow-0.15.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.15.0
+Version: 0.15.1
 Summary: HydraFlow seamlessly integrates Hydra and MLflow to streamline ML experiment management, combining Hydra's configuration management with MLflow's tracking capabilities.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

{hydraflow-0.15.0.dist-info → hydraflow-0.15.1.dist-info}/RECORD

@@ -5,17 +5,17 @@ hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/core/context.py,sha256=LFPNJxmuJQ2VUt-WBU07MC3ySbjlY8rRZ8VxuAih4o4,4148
 hydraflow/core/io.py,sha256=ZBXIL_jlBUiCI0L_J6S5S4OwtBMvdVVMXnekzMuC_JA,4404
 hydraflow/core/main.py,sha256=b9o6Rpn3uoXfDB8o0XZdl-g1yX2SKkOT12-H7lB8Les,5158
-hydraflow/core/run.py,sha256=9JNk3axDdKLpttGx-BC9aqw3d7rosygn2cIzL-fxVlM,11876
+hydraflow/core/run.py,sha256=KqaMdRUBOzOU4vkrRUczCrPCsVx30-XUQ_e78B78BSU,12330
 hydraflow/core/run_collection.py,sha256=pV3N83uBhmda9OeaNz1jqpF9z6A9j3jfUHtqy-uxCs4,15671
 hydraflow/core/run_info.py,sha256=3dW9GgWnZZNwbXwMrw-85AqQ956zlQddUi9irSNLR5g,2550
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hydraflow/executor/aio.py,sha256=xXsmBPIPdBlopv_1h0FdtOvoKUcuW7PQeKCV2d_lN9I,2122
-hydraflow/executor/conf.py,sha256=icGbLDh86KgkyiGXwDoEkmZpgAP3X8Jmu_PYqJoTooY,423
-hydraflow/executor/io.py,sha256=yZMcBVmAbPZZ82cAXhgiJfj9p8WvHmzOCMBg_vtEVek,1509
-hydraflow/executor/job.py,sha256=JX6xX9ffvHB7IiAVIfzVRjjnWKaPDxBgqdZf4ZO14CY,4651
-hydraflow/executor/parser.py,sha256=_Rfund3FDgrXitTt_znsTpgEtMDqZ_ICynaB_Zje14Q,14561
-hydraflow-0.15.0.dist-info/METADATA,sha256=2OpqrXDfnVxQ_ZJkS5tEjQH0VTa3yx8jkfFOjbkCK50,7238
-hydraflow-0.15.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.15.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.15.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.15.0.dist-info/RECORD,,
+hydraflow/executor/conf.py,sha256=8Xq4UAenRKJIl1NBgNbSfv6VUTJhdwPLayZIEAsiBR0,414
+hydraflow/executor/io.py,sha256=18wnHpCMQRGYL-oN2841h9W2aSW_X2SmO68Lx-3FIbU,1043
+hydraflow/executor/job.py,sha256=6QeJ18OMeocXeM04rCYL46GgArfX1SvZs9_4HTomTgE,5436
+hydraflow/executor/parser.py,sha256=RxP8qpDaJ8VLqZ51VlPFyVitWctObhkE_3iPIsY66Cs,14610
+hydraflow-0.15.1.dist-info/METADATA,sha256=oC-UgH0sZKw2Ry1kBiMPpNobxzlLhmhQgS8W3TIvGJI,7238
+hydraflow-0.15.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.15.1.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.15.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.15.1.dist-info/RECORD,,